From 61ed2785ac67349f68141404d61f69398f46f93d Mon Sep 17 00:00:00 2001 From: Dale John McGrew Date: Mon, 16 Apr 2018 13:41:56 -0700 Subject: [PATCH] Testing crashing fix copying ballot items. Better support for "Retrieve Possible Candidates from Google". --- ballot/models.py | 11 ++++- candidate/views_admin.py | 21 ++++++--- google_custom_search/controllers.py | 7 +++ google_custom_search/views_admin.py | 57 +++++++++++++++++-------- import_export_wikipedia/controllers.py | 3 ++ templates/candidate/candidate_list.html | 13 +++--- twitter/views_admin.py | 11 +++-- wevote_settings/models.py | 26 +++++------ 8 files changed, 103 insertions(+), 46 deletions(-) diff --git a/ballot/models.py b/ballot/models.py index 57c0157f5..f9e8d9c7d 100644 --- a/ballot/models.py +++ b/ballot/models.py @@ -2342,7 +2342,6 @@ def update_or_create_voter_ballot_saved( 'voter_id': voter_id, 'google_civic_election_id': google_civic_election_id, 'state_code': state_code, - 'election_date': election_day_text, 'election_description_text': election_description_text, 'original_text_for_map_search': original_text_for_map_search, 'substituted_address_nearby': substituted_address_nearby, @@ -2353,6 +2352,8 @@ def update_or_create_voter_ballot_saved( 'ballot_location_shortcut': ballot_location_shortcut, 'polling_location_we_vote_id_source': polling_location_we_vote_id_source, } + if positive_value_exists(election_day_text): + defaults['election_date'] = election_day_text if positive_value_exists(voter_id) and positive_value_exists(ballot_returned_we_vote_id): status += "SAVING_WITH_VOTER_ID_AND_BALLOT_RETURNED_WE_VOTE_ID " @@ -2453,7 +2454,9 @@ def copy_existing_ballot_items_from_stored_ballot(voter_id, text_for_map_search, if positive_value_exists(ballot_returned_we_vote_id): find_results = ballot_returned_manager.retrieve_ballot_returned_from_ballot_returned_we_vote_id( ballot_returned_we_vote_id) + status += "CALLING-RETRIEVE_BALLOT_RETURNED_FROM_WE_VOTE_ID, status: [[" status += find_results['status'] + status += "]] " if not find_results['ballot_returned_found']: error_results = { @@ -2478,7 +2481,9 @@ def copy_existing_ballot_items_from_stored_ballot(voter_id, text_for_map_search, elif positive_value_exists(ballot_location_shortcut): find_results = ballot_returned_manager.retrieve_ballot_returned_from_ballot_location_shortcut( ballot_location_shortcut) + status += "CALLING-RETRIEVE_BALLOT_RETURNED_FROM_BALLOT_LOCATION_SHORTCUT, status: [[" status += find_results['status'] + status += "]] " if not find_results['ballot_returned_found']: error_results = { @@ -2503,7 +2508,9 @@ def copy_existing_ballot_items_from_stored_ballot(voter_id, text_for_map_search, elif positive_value_exists(google_civic_election_id) and text_for_map_search_empty: find_results = ballot_returned_manager.retrieve_ballot_returned_from_google_civic_election_id( google_civic_election_id) + status += "CALLING-RETRIEVE_BALLOT_RETURNED_FROM_GOOGLE_CIVIC_ELECTION_ID, status: [[" status += find_results['status'] + status += "]] " if not find_results['ballot_returned_found']: error_results = { @@ -2528,7 +2535,9 @@ def copy_existing_ballot_items_from_stored_ballot(voter_id, text_for_map_search, else: find_results = ballot_returned_manager.find_closest_ballot_returned( text_for_map_search, google_civic_election_id) + status += "CALLING-FIND_CLOSEST_BALLOT_RETURNED, status: [[" status += find_results['status'] + status += "]] " if not find_results['ballot_returned_found']: error_results = { diff --git a/candidate/views_admin.py b/candidate/views_admin.py index eff19c8ca..bbfa696fe 100644 --- a/candidate/views_admin.py +++ b/candidate/views_admin.py @@ -35,7 +35,8 @@ import wevote_functions.admin from wevote_functions.functions import convert_to_int, extract_twitter_handle_from_text_string, \ positive_value_exists, STATE_CODE_MAP -from wevote_settings.models import RemoteRequestHistory +from wevote_settings.models import RemoteRequestHistory, \ + RETRIEVE_POSSIBLE_GOOGLE_LINKS, RETRIEVE_POSSIBLE_TWITTER_HANDLES from django.http import HttpResponse import json @@ -295,9 +296,10 @@ def candidate_list_view(request): if twitter_possibility_list and positive_value_exists(len(twitter_possibility_list)): candidate.candidate_merge_possibility = twitter_possibility_list[0] else: - request_history = RemoteRequestHistory.objects.filter( - candidate_campaign_we_vote_id__iexact=candidate.we_vote_id) - request_history_list = list(request_history) + request_history_query = RemoteRequestHistory.objects.filter( + candidate_campaign_we_vote_id__iexact=candidate.we_vote_id, + kind_of_action=RETRIEVE_POSSIBLE_TWITTER_HANDLES) + request_history_list = list(request_history_query) if request_history_list and positive_value_exists(len(request_history_list)): candidate.no_twitter_possibilities_found = True except Exception as e: @@ -311,7 +313,16 @@ def candidate_list_view(request): exclude(item_image__isnull=True).exclude(item_image__exact='') google_search_possibility_query = google_search_possibility_query.order_by( '-chosen_and_updated', 'not_a_match', '-likelihood_score') - candidate.google_search_merge_possibility = google_search_possibility_query[0] + google_search_merge_possibility = list(google_search_possibility_query) + if google_search_merge_possibility and positive_value_exists(len(google_search_merge_possibility)): + candidate.google_search_merge_possibility = google_search_possibility_query[0] + else: + request_history_query = RemoteRequestHistory.objects.filter( + candidate_campaign_we_vote_id__iexact=candidate.we_vote_id, + kind_of_action=RETRIEVE_POSSIBLE_GOOGLE_LINKS) + request_history_list = list(request_history_query) + if request_history_list and positive_value_exists(len(request_history_list)): + candidate.no_google_possibilities_found = True except Exception as e: candidate.google_search_merge_possibility = None diff --git a/google_custom_search/controllers.py b/google_custom_search/controllers.py index 2abec4636..45e20fd25 100644 --- a/google_custom_search/controllers.py +++ b/google_custom_search/controllers.py @@ -14,6 +14,7 @@ from re import sub from wevote_functions.functions import positive_value_exists, convert_state_code_to_state_text, \ POSITIVE_SEARCH_KEYWORDS, NEGATIVE_SEARCH_KEYWORDS, extract_facebook_username_from_text_string +from wevote_settings.models import RemoteRequestHistoryManager, RETRIEVE_POSSIBLE_GOOGLE_LINKS def delete_possible_google_search_users(candidate_campaign): @@ -215,6 +216,12 @@ def retrieve_possible_google_search_users(candidate_campaign, voter_device_id): if google_search_user_count == MAXIMUM_GOOGLE_SEARCH_USERS: break + # Create a record denoting that we have retrieved from Google for this candidate + remote_request_history_manager = RemoteRequestHistoryManager() + save_results_history = remote_request_history_manager.create_remote_request_history_entry( + RETRIEVE_POSSIBLE_GOOGLE_LINKS, candidate_campaign.google_civic_election_id, + candidate_campaign.we_vote_id, None, len(possible_google_search_users_list), status) + results = { 'success': True, 'status': status, diff --git a/google_custom_search/views_admin.py b/google_custom_search/views_admin.py index 61ec176e2..50c53fb05 100644 --- a/google_custom_search/views_admin.py +++ b/google_custom_search/views_admin.py @@ -14,6 +14,7 @@ from voter.models import voter_has_authority from wevote_functions.functions import convert_to_int, positive_value_exists, get_voter_api_device_id import wevote_functions.admin +from wevote_settings.models import RemoteRequestHistory, RETRIEVE_POSSIBLE_GOOGLE_LINKS logger = wevote_functions.admin.get_logger(__name__) @@ -112,6 +113,18 @@ def bulk_retrieve_possible_google_search_users_view(request): page = request.GET.get('page', 0) state_code = request.GET.get('state_code', '') show_all = request.GET.get('show_all', False) + limit = convert_to_int(request.GET.get('show_all', 0)) + + if not positive_value_exists(google_civic_election_id) and not positive_value_exists(state_code) \ + and not positive_value_exists(limit): + messages.add_message(request, messages.ERROR, + 'bulk_retrieve_possible_google_search_users_view, LIMITING_VARIABLE_REQUIRED') + return HttpResponseRedirect(reverse('candidate:candidate_list', args=()) + + '?google_civic_election_id=' + str(google_civic_election_id) + + '&state_code=' + str(state_code) + + '&hide_candidate_tools=' + str(hide_candidate_tools) + + '&page=' + str(page) + ) try: candidate_list = CandidateCampaign.objects.all() @@ -120,8 +133,8 @@ def bulk_retrieve_possible_google_search_users_view(request): if positive_value_exists(state_code): candidate_list = candidate_list.filter(state_code__iexact=state_code) candidate_list = candidate_list.order_by('candidate_name') - if not positive_value_exists(show_all): - candidate_list = candidate_list[:200] + if positive_value_exists(limit): + candidate_list = candidate_list[:limit] candidate_list_count = candidate_list.count() # Run google search and analysis on candidates without a linked or possible google search @@ -130,21 +143,31 @@ def bulk_retrieve_possible_google_search_users_view(request): while positive_value_exists(number_of_candidates_to_search) \ and (current_candidate_index < candidate_list_count): one_candidate = candidate_list[current_candidate_index] - google_search_possibility_list = [] - try: - google_search_possibility_query = GoogleSearchUser.objects.filter( - candidate_campaign_we_vote_id=one_candidate.we_vote_id) - google_search_possibility_query = google_search_possibility_query.order_by( - '-chosen_and_updated', '-likelihood_score') - google_search_possibility_list = list(google_search_possibility_query) - except Exception as e: - pass - - if not positive_value_exists(google_search_possibility_list): - # Google search and analysis has not been run on this candidate yet - # (or no results have been found for this candidate, at least) - results = retrieve_possible_google_search_users(one_candidate, voter_device_id) - number_of_candidates_to_search -= 1 + if not positive_value_exists(one_candidate.candidate_twitter_handle): + # Candidate does not have a Twitter account linked - only search for these + # Check to see if we have already tried to find their information from Twitter. We don't want to + # search Twitter more than once. + request_history_query = RemoteRequestHistory.objects.filter( + candidate_campaign_we_vote_id__iexact=one_candidate.we_vote_id, + kind_of_action=RETRIEVE_POSSIBLE_GOOGLE_LINKS) + request_history_list = list(request_history_query) + + if not positive_value_exists(request_history_list): + google_search_possibility_list = [] + try: + google_search_possibility_query = GoogleSearchUser.objects.filter( + candidate_campaign_we_vote_id=one_candidate.we_vote_id) + google_search_possibility_query = google_search_possibility_query.order_by( + '-chosen_and_updated', '-likelihood_score') + google_search_possibility_list = list(google_search_possibility_query) + except Exception as e: + pass + + if not positive_value_exists(google_search_possibility_list): + # Google search and analysis has not been run on this candidate yet + # (or no results have been found for this candidate, at least) + results = retrieve_possible_google_search_users(one_candidate, voter_device_id) + number_of_candidates_to_search -= 1 current_candidate_index += 1 except CandidateCampaign.DoesNotExist: # This is fine, do nothing diff --git a/import_export_wikipedia/controllers.py b/import_export_wikipedia/controllers.py index 9842ca7c6..8f599e9d2 100644 --- a/import_export_wikipedia/controllers.py +++ b/import_export_wikipedia/controllers.py @@ -297,6 +297,9 @@ def reach_out_to_wikipedia_with_guess(wikipedia_page_title_guess, auto_suggest=F # There are a few possible pages this might refer to status = 'WIKIPEDIA_DISAMBIGUATION_ERROR ' page_found = False + except Exception as e: + status = "WIKIPEDIA_UNKNOWN_ERROR " + page_found = False results = { 'success': page_found, diff --git a/templates/candidate/candidate_list.html b/templates/candidate/candidate_list.html index 5a9e51726..12a083778 100644 --- a/templates/candidate/candidate_list.html +++ b/templates/candidate/candidate_list.html @@ -89,8 +89,8 @@

Candidates

  • Retrieve Possible Twitter Handles (25 at a time - about 30 seconds - Open in New Window)
    • -
  • - Retrieve Possible Candidates from Google (20 at a time - 1-2 minutes)
    • +
  • + Retrieve Possible Candidates from Google (20 at a time - 1-2 minutes - Open in New Window)
  • Retrieve/Refresh Candidate Twitter Info for this Election (1-2 minutes)
    • @@ -217,9 +217,8 @@

    Candidates

    {% if not candidate.google_search_merge_possibility.chosen_and_updated %}   - (Best Guess: - - {{ candidate.google_search_merge_possibility.likelihood_score }})
    + ( + Best Guess: {{ candidate.google_search_merge_possibility.likelihood_score }})
    {{ candidate.google_search_merge_possibility.item_link|default_if_none:"" }}
    {{ candidate.google_search_merge_possibility.item_snippet }} {% if candidate.google_search_merge_possibility.likelihood_score >= 50 %} @@ -239,7 +238,9 @@

    Candidates

      {{ candidate.google_search_merge_possibility.item_link|default_if_none:"" }}
    {% endif %} - + {% elif candidate.no_google_possibilities_found %} + {% endif %} {% if candidate.candidate_url %}{{ candidate.candidate_url }}{% endif %} diff --git a/twitter/views_admin.py b/twitter/views_admin.py index 9d6f4afd0..c8ac0e652 100644 --- a/twitter/views_admin.py +++ b/twitter/views_admin.py @@ -13,7 +13,7 @@ from voter.models import voter_has_authority from wevote_functions.functions import convert_to_int, positive_value_exists import wevote_functions.admin -from wevote_settings.models import RemoteRequestHistory +from wevote_settings.models import RemoteRequestHistory, RETRIEVE_POSSIBLE_TWITTER_HANDLES logger = wevote_functions.admin.get_logger(__name__) @@ -104,9 +104,12 @@ def bulk_retrieve_possible_twitter_handles_view(request): one_candidate = candidate_list[current_candidate_index] if not positive_value_exists(one_candidate.candidate_twitter_handle): # Candidate does not have a Twitter account linked - request_history = RemoteRequestHistory.objects.filter( - candidate_campaign_we_vote_id__iexact=one_candidate.we_vote_id) - request_history_list = list(request_history) + # Check to see if we have already tried to find their information from Twitter. We don't want to + # search Twitter more than once. + request_history_query = RemoteRequestHistory.objects.filter( + candidate_campaign_we_vote_id__iexact=one_candidate.we_vote_id, + kind_of_action=RETRIEVE_POSSIBLE_TWITTER_HANDLES) + request_history_list = list(request_history_query) if not positive_value_exists(request_history_list): # Twitter account search and analysis has not been run on this candidate yet diff --git a/wevote_settings/models.py b/wevote_settings/models.py index 32f7e606e..bdb6ddfd4 100644 --- a/wevote_settings/models.py +++ b/wevote_settings/models.py @@ -10,7 +10,15 @@ from wevote_functions.functions import convert_to_int, generate_random_string, positive_value_exists +RETRIEVE_POSSIBLE_GOOGLE_LINKS = 'RETRIEVE_POSSIBLE_GOOGLE_LINKS' RETRIEVE_POSSIBLE_TWITTER_HANDLES = 'RETRIEVE_POSSIBLE_TWITTER_HANDLES' +STOP_BULK_SEARCH_TWITTER_LINK_POSSIBILITY = 'STOP_BULK_SEARCH_TWITTER_LINK_POSSIBILITY' + +KIND_OF_ACTION_CHOICES = ( + (RETRIEVE_POSSIBLE_GOOGLE_LINKS, 'Retrieve possible google links'), + (RETRIEVE_POSSIBLE_TWITTER_HANDLES, 'Retrieve possible twitter handles'), + (STOP_BULK_SEARCH_TWITTER_LINK_POSSIBILITY, 'Stop search for Bulk twitter links'), +) logger = wevote_functions.admin.get_logger(__name__) @@ -254,16 +262,6 @@ def fetch_next_we_vote_id_electoral_district_integer(): def fetch_next_we_vote_id_party_integer(): return fetch_next_we_vote_id_integer('we_vote_id_party_integer') -SEARCH_TWITTER_LINK_POSSIBILITY = 'SEARCH_TWITTER_LINK_POSSIBILITY' -SEARCH_GOOGLE_LINK_POSSIBILITY = 'SEARCH_GOOGLE_LINK_POSSIBILITY' -STOP_BULK_SEARCH_TWITTER_LINK_POSSIBILITY = 'STOP_BULK_SEARCH_TWITTER_LINK_POSSIBILITY' - -KIND_OF_ACTION_CHOICES = ( - (SEARCH_TWITTER_LINK_POSSIBILITY, 'Retrieve possible twitter handles'), - (SEARCH_GOOGLE_LINK_POSSIBILITY, 'Retrieve possible google links'), - (STOP_BULK_SEARCH_TWITTER_LINK_POSSIBILITY, 'Stop search for Bulk twitter links'), -) - class RemoteRequestHistory(models.Model): """ @@ -275,10 +273,12 @@ class RemoteRequestHistory(models.Model): google_civic_election_id = models.PositiveIntegerField(verbose_name="google civic election id", null=True) kind_of_action = models.CharField(verbose_name="kind of action to take", max_length=50, - choices=KIND_OF_ACTION_CHOICES, default=SEARCH_TWITTER_LINK_POSSIBILITY) - candidate_campaign_we_vote_id = models.CharField(verbose_name="candidate we vote id", max_length=255, unique=False, null=True) + choices=KIND_OF_ACTION_CHOICES, null=True) + candidate_campaign_we_vote_id = models.CharField(verbose_name="candidate we vote id", max_length=255, unique=False, + null=True) - organization_we_vote_id = models.CharField(verbose_name="we vote id for the org owner", max_length=255, unique=False, null=True) + organization_we_vote_id = models.CharField(verbose_name="we vote id for the org owner", max_length=255, + unique=False, null=True) number_of_results = models.PositiveIntegerField(verbose_name="number of results", null=True, default=0) status = models.CharField(verbose_name="Request status message", max_length=255, default="", null=True, blank=True)