Testing crashing fix copying ballot items. Better support for "Retrie…

…ve Possible Candidates from Google".

ballot/models.py

@@ -2342,7 +2342,6 @@ def update_or_create_voter_ballot_saved(
                 'voter_id': voter_id,
                 'google_civic_election_id': google_civic_election_id,
                 'state_code': state_code,
-                'election_date': election_day_text,
                 'election_description_text': election_description_text,
                 'original_text_for_map_search': original_text_for_map_search,
                 'substituted_address_nearby': substituted_address_nearby,
@@ -2353,6 +2352,8 @@ def update_or_create_voter_ballot_saved(
                 'ballot_location_shortcut': ballot_location_shortcut,
                 'polling_location_we_vote_id_source': polling_location_we_vote_id_source,
             }
+            if positive_value_exists(election_day_text):
+                defaults['election_date'] = election_day_text
 
             if positive_value_exists(voter_id) and positive_value_exists(ballot_returned_we_vote_id):
                 status += "SAVING_WITH_VOTER_ID_AND_BALLOT_RETURNED_WE_VOTE_ID "
@@ -2453,7 +2454,9 @@ def copy_existing_ballot_items_from_stored_ballot(voter_id, text_for_map_search,
     if positive_value_exists(ballot_returned_we_vote_id):
         find_results = ballot_returned_manager.retrieve_ballot_returned_from_ballot_returned_we_vote_id(
             ballot_returned_we_vote_id)
+        status += "CALLING-RETRIEVE_BALLOT_RETURNED_FROM_WE_VOTE_ID, status: [["
         status += find_results['status']
+        status += "]] "
 
         if not find_results['ballot_returned_found']:
             error_results = {
@@ -2478,7 +2481,9 @@ def copy_existing_ballot_items_from_stored_ballot(voter_id, text_for_map_search,
     elif positive_value_exists(ballot_location_shortcut):
         find_results = ballot_returned_manager.retrieve_ballot_returned_from_ballot_location_shortcut(
             ballot_location_shortcut)
+        status += "CALLING-RETRIEVE_BALLOT_RETURNED_FROM_BALLOT_LOCATION_SHORTCUT, status: [["
         status += find_results['status']
+        status += "]] "
 
         if not find_results['ballot_returned_found']:
             error_results = {
@@ -2503,7 +2508,9 @@ def copy_existing_ballot_items_from_stored_ballot(voter_id, text_for_map_search,
     elif positive_value_exists(google_civic_election_id) and text_for_map_search_empty:
         find_results = ballot_returned_manager.retrieve_ballot_returned_from_google_civic_election_id(
             google_civic_election_id)
+        status += "CALLING-RETRIEVE_BALLOT_RETURNED_FROM_GOOGLE_CIVIC_ELECTION_ID, status: [["
         status += find_results['status']
+        status += "]] "
 
         if not find_results['ballot_returned_found']:
             error_results = {
@@ -2528,7 +2535,9 @@ def copy_existing_ballot_items_from_stored_ballot(voter_id, text_for_map_search,
     else:
         find_results = ballot_returned_manager.find_closest_ballot_returned(
             text_for_map_search, google_civic_election_id)
+        status += "CALLING-FIND_CLOSEST_BALLOT_RETURNED, status: [["
         status += find_results['status']
+        status += "]] "
 
         if not find_results['ballot_returned_found']:
             error_results = {

candidate/views_admin.py

@@ -35,7 +35,8 @@
 import wevote_functions.admin
 from wevote_functions.functions import convert_to_int, extract_twitter_handle_from_text_string, \
     positive_value_exists, STATE_CODE_MAP
-from wevote_settings.models import RemoteRequestHistory
+from wevote_settings.models import RemoteRequestHistory, \
+    RETRIEVE_POSSIBLE_GOOGLE_LINKS, RETRIEVE_POSSIBLE_TWITTER_HANDLES
 from django.http import HttpResponse
 import json
 
@@ -295,9 +296,10 @@ def candidate_list_view(request):
             if twitter_possibility_list and positive_value_exists(len(twitter_possibility_list)):
                 candidate.candidate_merge_possibility = twitter_possibility_list[0]
             else:
-                request_history = RemoteRequestHistory.objects.filter(
-                    candidate_campaign_we_vote_id__iexact=candidate.we_vote_id)
-                request_history_list = list(request_history)
+                request_history_query = RemoteRequestHistory.objects.filter(
+                    candidate_campaign_we_vote_id__iexact=candidate.we_vote_id,
+                    kind_of_action=RETRIEVE_POSSIBLE_TWITTER_HANDLES)
+                request_history_list = list(request_history_query)
                 if request_history_list and positive_value_exists(len(request_history_list)):
                     candidate.no_twitter_possibilities_found = True
         except Exception as e:
@@ -311,7 +313,16 @@ def candidate_list_view(request):
                 exclude(item_image__isnull=True).exclude(item_image__exact='')
             google_search_possibility_query = google_search_possibility_query.order_by(
                 '-chosen_and_updated', 'not_a_match', '-likelihood_score')
-            candidate.google_search_merge_possibility = google_search_possibility_query[0]
+            google_search_merge_possibility = list(google_search_possibility_query)
+            if google_search_merge_possibility and positive_value_exists(len(google_search_merge_possibility)):
+                candidate.google_search_merge_possibility = google_search_possibility_query[0]
+            else:
+                request_history_query = RemoteRequestHistory.objects.filter(
+                    candidate_campaign_we_vote_id__iexact=candidate.we_vote_id,
+                    kind_of_action=RETRIEVE_POSSIBLE_GOOGLE_LINKS)
+                request_history_list = list(request_history_query)
+                if request_history_list and positive_value_exists(len(request_history_list)):
+                    candidate.no_google_possibilities_found = True
         except Exception as e:
             candidate.google_search_merge_possibility = None
 

google_custom_search/controllers.py

@@ -14,6 +14,7 @@
 from re import sub
 from wevote_functions.functions import positive_value_exists, convert_state_code_to_state_text, \
     POSITIVE_SEARCH_KEYWORDS, NEGATIVE_SEARCH_KEYWORDS, extract_facebook_username_from_text_string
+from wevote_settings.models import RemoteRequestHistoryManager, RETRIEVE_POSSIBLE_GOOGLE_LINKS
 
 
 def delete_possible_google_search_users(candidate_campaign):
@@ -215,6 +216,12 @@ def retrieve_possible_google_search_users(candidate_campaign, voter_device_id):
                 if google_search_user_count == MAXIMUM_GOOGLE_SEARCH_USERS:
                     break
 
+    # Create a record denoting that we have retrieved from Google for this candidate
+    remote_request_history_manager = RemoteRequestHistoryManager()
+    save_results_history = remote_request_history_manager.create_remote_request_history_entry(
+        RETRIEVE_POSSIBLE_GOOGLE_LINKS, candidate_campaign.google_civic_election_id,
+        candidate_campaign.we_vote_id, None, len(possible_google_search_users_list), status)
+
     results = {
         'success':                  True,
         'status':                   status,

google_custom_search/views_admin.py

@@ -14,6 +14,7 @@
 from voter.models import voter_has_authority
 from wevote_functions.functions import convert_to_int, positive_value_exists, get_voter_api_device_id
 import wevote_functions.admin
+from wevote_settings.models import RemoteRequestHistory, RETRIEVE_POSSIBLE_GOOGLE_LINKS
 
 logger = wevote_functions.admin.get_logger(__name__)
 
@@ -112,6 +113,18 @@ def bulk_retrieve_possible_google_search_users_view(request):
     page = request.GET.get('page', 0)
     state_code = request.GET.get('state_code', '')
     show_all = request.GET.get('show_all', False)
+    limit = convert_to_int(request.GET.get('show_all', 0))
+
+    if not positive_value_exists(google_civic_election_id) and not positive_value_exists(state_code) \
+            and not positive_value_exists(limit):
+        messages.add_message(request, messages.ERROR,
+                             'bulk_retrieve_possible_google_search_users_view, LIMITING_VARIABLE_REQUIRED')
+        return HttpResponseRedirect(reverse('candidate:candidate_list', args=()) +
+                                    '?google_civic_election_id=' + str(google_civic_election_id) +
+                                    '&state_code=' + str(state_code) +
+                                    '&hide_candidate_tools=' + str(hide_candidate_tools) +
+                                    '&page=' + str(page)
+                                    )
 
     try:
         candidate_list = CandidateCampaign.objects.all()
@@ -120,8 +133,8 @@ def bulk_retrieve_possible_google_search_users_view(request):
         if positive_value_exists(state_code):
             candidate_list = candidate_list.filter(state_code__iexact=state_code)
         candidate_list = candidate_list.order_by('candidate_name')
-        if not positive_value_exists(show_all):
-            candidate_list = candidate_list[:200]
+        if positive_value_exists(limit):
+            candidate_list = candidate_list[:limit]
         candidate_list_count = candidate_list.count()
 
         # Run google search and analysis on candidates without a linked or possible google search
@@ -130,21 +143,31 @@ def bulk_retrieve_possible_google_search_users_view(request):
         while positive_value_exists(number_of_candidates_to_search) \
                 and (current_candidate_index < candidate_list_count):
             one_candidate = candidate_list[current_candidate_index]
-            google_search_possibility_list = []
-            try:
-                google_search_possibility_query = GoogleSearchUser.objects.filter(
-                    candidate_campaign_we_vote_id=one_candidate.we_vote_id)
-                google_search_possibility_query = google_search_possibility_query.order_by(
-                    '-chosen_and_updated', '-likelihood_score')
-                google_search_possibility_list = list(google_search_possibility_query)
-            except Exception as e:
-                pass
-
-            if not positive_value_exists(google_search_possibility_list):
-                # Google search and analysis has not been run on this candidate yet
-                # (or no results have been found for this candidate, at least)
-                results = retrieve_possible_google_search_users(one_candidate, voter_device_id)
-                number_of_candidates_to_search -= 1
+            if not positive_value_exists(one_candidate.candidate_twitter_handle):
+                # Candidate does not have a Twitter account linked - only search for these
+                # Check to see if we have already tried to find their information from Twitter. We don't want to
+                #  search Twitter more than once.
+                request_history_query = RemoteRequestHistory.objects.filter(
+                    candidate_campaign_we_vote_id__iexact=one_candidate.we_vote_id,
+                    kind_of_action=RETRIEVE_POSSIBLE_GOOGLE_LINKS)
+                request_history_list = list(request_history_query)
+
+                if not positive_value_exists(request_history_list):
+                    google_search_possibility_list = []
+                    try:
+                        google_search_possibility_query = GoogleSearchUser.objects.filter(
+                            candidate_campaign_we_vote_id=one_candidate.we_vote_id)
+                        google_search_possibility_query = google_search_possibility_query.order_by(
+                            '-chosen_and_updated', '-likelihood_score')
+                        google_search_possibility_list = list(google_search_possibility_query)
+                    except Exception as e:
+                        pass
+
+                    if not positive_value_exists(google_search_possibility_list):
+                        # Google search and analysis has not been run on this candidate yet
+                        # (or no results have been found for this candidate, at least)
+                        results = retrieve_possible_google_search_users(one_candidate, voter_device_id)
+                        number_of_candidates_to_search -= 1
             current_candidate_index += 1
     except CandidateCampaign.DoesNotExist:
         # This is fine, do nothing

import_export_wikipedia/controllers.py

@@ -297,6 +297,9 @@ def reach_out_to_wikipedia_with_guess(wikipedia_page_title_guess, auto_suggest=F
         # There are a few possible pages this might refer to
         status = 'WIKIPEDIA_DISAMBIGUATION_ERROR '
         page_found = False
+    except Exception as e:
+        status = "WIKIPEDIA_UNKNOWN_ERROR "
+        page_found = False
 
     results = {
         'success':          page_found,

templates/candidate/candidate_list.html

@@ -89,8 +89,8 @@ <h1>Candidates</h1>
         <li><a href="{% url 'twitter2:bulk_retrieve_possible_twitter_handles' %}?google_civic_election_id={{ google_civic_election_id }}&state_code={{ state_code }}&page={{ current_page_number }}" target="_blank" >
         Retrieve Possible Twitter Handles</a> (25 at a time - about 30 seconds - Open in New Window)</li>
 
-        <li><a href="{% url 'google_custom_search:bulk_retrieve_possible_google_search_users' %}?google_civic_election_id={{ google_civic_election_id }}&state_code={{ state_code }}" >
-        Retrieve Possible Candidates from Google</a> (20 at a time - 1-2 minutes)</li>
+        <li><a href="{% url 'google_custom_search:bulk_retrieve_possible_google_search_users' %}?google_civic_election_id={{ google_civic_election_id }}&state_code={{ state_code }}" target="_blank" >
+        Retrieve Possible Candidates from Google</a> (20 at a time - 1-2 minutes - Open in New Window)</li>
 
         <li><a href="{% url 'import_export_twitter:refresh_twitter_candidate_details_for_election' google_civic_election_id %}?state_code={{ state_code }}" >
         Retrieve/Refresh Candidate Twitter Info for this Election</a> (1-2 minutes)</li>
@@ -217,9 +217,8 @@ <h1>Candidates</h1>
                     {% if not candidate.google_search_merge_possibility.chosen_and_updated %}
                         <a href="{% url 'candidate:candidate_edit' candidate.id %}?google_civic_election_id={{ google_civic_election_id }}&state_code={{ state_code }}">
                             <img src='{{ candidate.google_search_merge_possibility.item_image }}' height="25px" /></a>&nbsp;
-                        (Best Guess:
-                        <a href="{% url 'candidate:candidate_edit' candidate.id %}?google_civic_election_id={{ google_civic_election_id }}&state_code={{ state_code }}">
-                            {{ candidate.google_search_merge_possibility.likelihood_score }}</a>)<br />
+                        (<a href="{% url 'candidate:candidate_edit' candidate.id %}?google_civic_election_id={{ google_civic_election_id }}&state_code={{ state_code }}&show_all_google_search_users=1">
+                            Best Guess: {{ candidate.google_search_merge_possibility.likelihood_score }}</a>)<br />
                         <a href="{{ candidate.google_search_merge_possibility.item_link }}" target="_blank">{{ candidate.google_search_merge_possibility.item_link|default_if_none:"" }}</a><br />
                         {{ candidate.google_search_merge_possibility.item_snippet }}
                         {% if candidate.google_search_merge_possibility.likelihood_score >= 50 %}
@@ -239,7 +238,9 @@ <h1>Candidates</h1>
                             <img src='{{ candidate.google_search_merge_possibility.item_image }}' height="25px" /></a>&nbsp;
                         <a href="{{ candidate.google_search_merge_possibility.item_link }}" target="_blank">{{ candidate.google_search_merge_possibility.item_link|default_if_none:"" }}</a><br />
                     {% endif %}
-
+                {% elif candidate.no_google_possibilities_found %}
+                    <ul><li>no options found
+                    </li></ul>
                 {% endif %}
             </td>
             <td>{% if candidate.candidate_url %}<a href="{{ candidate.candidate_url }}" target="_blank">{{ candidate.candidate_url }}</a>{% endif %}</td>

twitter/views_admin.py

@@ -13,7 +13,7 @@
 from voter.models import voter_has_authority
 from wevote_functions.functions import convert_to_int, positive_value_exists
 import wevote_functions.admin
-from wevote_settings.models import RemoteRequestHistory
+from wevote_settings.models import RemoteRequestHistory, RETRIEVE_POSSIBLE_TWITTER_HANDLES
 
 logger = wevote_functions.admin.get_logger(__name__)
 
@@ -104,9 +104,12 @@ def bulk_retrieve_possible_twitter_handles_view(request):
             one_candidate = candidate_list[current_candidate_index]
             if not positive_value_exists(one_candidate.candidate_twitter_handle):
                 # Candidate does not have a Twitter account linked
-                request_history = RemoteRequestHistory.objects.filter(
-                    candidate_campaign_we_vote_id__iexact=one_candidate.we_vote_id)
-                request_history_list = list(request_history)
+                # Check to see if we have already tried to find their information from Twitter. We don't want to
+                #  search Twitter more than once.
+                request_history_query = RemoteRequestHistory.objects.filter(
+                    candidate_campaign_we_vote_id__iexact=one_candidate.we_vote_id,
+                    kind_of_action=RETRIEVE_POSSIBLE_TWITTER_HANDLES)
+                request_history_list = list(request_history_query)
 
                 if not positive_value_exists(request_history_list):
                     # Twitter account search and analysis has not been run on this candidate yet

wevote_settings/models.py

@@ -10,7 +10,15 @@
 from wevote_functions.functions import convert_to_int, generate_random_string, positive_value_exists
 
 
+RETRIEVE_POSSIBLE_GOOGLE_LINKS = 'RETRIEVE_POSSIBLE_GOOGLE_LINKS'
 RETRIEVE_POSSIBLE_TWITTER_HANDLES = 'RETRIEVE_POSSIBLE_TWITTER_HANDLES'
+STOP_BULK_SEARCH_TWITTER_LINK_POSSIBILITY = 'STOP_BULK_SEARCH_TWITTER_LINK_POSSIBILITY'
+
+KIND_OF_ACTION_CHOICES = (
+    (RETRIEVE_POSSIBLE_GOOGLE_LINKS, 'Retrieve possible google links'),
+    (RETRIEVE_POSSIBLE_TWITTER_HANDLES, 'Retrieve possible twitter handles'),
+    (STOP_BULK_SEARCH_TWITTER_LINK_POSSIBILITY, 'Stop search for Bulk twitter links'),
+)
 
 logger = wevote_functions.admin.get_logger(__name__)
 
@@ -254,16 +262,6 @@ def fetch_next_we_vote_id_electoral_district_integer():
 def fetch_next_we_vote_id_party_integer():
     return fetch_next_we_vote_id_integer('we_vote_id_party_integer')
 
-SEARCH_TWITTER_LINK_POSSIBILITY = 'SEARCH_TWITTER_LINK_POSSIBILITY'
-SEARCH_GOOGLE_LINK_POSSIBILITY = 'SEARCH_GOOGLE_LINK_POSSIBILITY'
-STOP_BULK_SEARCH_TWITTER_LINK_POSSIBILITY = 'STOP_BULK_SEARCH_TWITTER_LINK_POSSIBILITY'
-
-KIND_OF_ACTION_CHOICES = (
-    (SEARCH_TWITTER_LINK_POSSIBILITY, 'Retrieve possible twitter handles'),
-    (SEARCH_GOOGLE_LINK_POSSIBILITY, 'Retrieve possible google links'),
-    (STOP_BULK_SEARCH_TWITTER_LINK_POSSIBILITY, 'Stop search for Bulk twitter links'),
-)
-
 
 class RemoteRequestHistory(models.Model):
     """
@@ -275,10 +273,12 @@ class RemoteRequestHistory(models.Model):
     google_civic_election_id = models.PositiveIntegerField(verbose_name="google civic election id", null=True)
 
     kind_of_action = models.CharField(verbose_name="kind of action to take", max_length=50,
-                                      choices=KIND_OF_ACTION_CHOICES, default=SEARCH_TWITTER_LINK_POSSIBILITY)
-    candidate_campaign_we_vote_id = models.CharField(verbose_name="candidate we vote id", max_length=255, unique=False, null=True)
+                                      choices=KIND_OF_ACTION_CHOICES, null=True)
+    candidate_campaign_we_vote_id = models.CharField(verbose_name="candidate we vote id", max_length=255, unique=False,
+                                                     null=True)
 
-    organization_we_vote_id = models.CharField(verbose_name="we vote id for the org owner", max_length=255, unique=False, null=True)
+    organization_we_vote_id = models.CharField(verbose_name="we vote id for the org owner", max_length=255,
+                                               unique=False, null=True)
     number_of_results = models.PositiveIntegerField(verbose_name="number of results", null=True, default=0)
     status = models.CharField(verbose_name="Request status message", max_length=255, default="", null=True, blank=True)