Merge pull request #1104 from DemocracyClub/exclude-candidacies-from-…

…import-ballots

Exclude candidacies from import ballots

sam-template.yaml

@@ -8,7 +8,7 @@ Description: >
 # More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst
 Globals:
   Function:
-    Timeout: 240
+    Timeout: 150
 
 Parameters:
 
@@ -101,10 +101,10 @@ Resources:
         ImportPeopleRecentlyUpdated:
           Type: Schedule # More info about API Event Source: https://github.com/aws/serverless-application-model/blob/master/versions/2016-10-31.md#schedule
           Properties:
-            Schedule: rate(5 minutes)
+            Schedule: cron(2-57/5 * ? * * *) # using the ? as you cannot use * wildcard for both day-of-month and day-of-week field. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html#CronExpressions
             Name: import-people-recently-updated
             Description: Update all people updated in YNR recently
-            Input: '{"command": "import_people", "args": ["--recently-updated"]}'
+            Input: '{"command": "import_people", "args": ["--recently-updated", "--exclude-candidacies"]}'
         DeleteDeletedPeople:
           Type: Schedule # More info about API Event Source: https://github.com/aws/serverless-application-model/blob/master/versions/2016-10-31.md#schedule
           Properties:
@@ -115,7 +115,7 @@ Resources:
         ImportBallotsRecentlyUpdated:
           Type: Schedule # More info about API Event Source: https://github.com/aws/serverless-application-model/blob/master/versions/2016-10-31.md#schedule
           Properties:
-            Schedule: rate(5 minutes)
+            Schedule: cron(*/5 * ? * * *) # using the ? as you cannot use * wildcard for both day-of-month and day-of-week field. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html#CronExpressions
             Name: import-ballots-recently-updated
             Description: Update all ballots updated in YNR recently
             Input: '{"command": "import_ballots", "args": ["--recently-updated"]}'
@@ -129,7 +129,7 @@ Resources:
         BatchFeedbackToSlack:
           Type: Schedule
           Properties:
-            Schedule: cron(0 9 ? * 6 *)
+            Schedule: cron(0 9 ? * * *) # using the ? as you cannot use * wildcard for both day-of-month and day-of-week field. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html#CronExpressions
             Name: batch-feedback-to-slack
             Description: Send feedback entries for the last week to slack
             Input: '{"command": "batch_feedback_to_slack", "args": ["--hours=168"]}'

wcivf/apps/api/views.py

@@ -4,14 +4,17 @@
 from rest_framework.exceptions import APIException
 from rest_framework.response import Response
 from rest_framework.views import APIView
+
+from django.conf import settings
+from django.utils.http import urlencode
+
 from api import serializers
 from api.serializers import VotingSystemSerializer
 from core.helpers import clean_postcode
-from hustings.api.serializers import HustingSerializer
-
-from people.models import Person
 from elections.views import mixins
 from elections.models import PostElection, InvalidPostcodeError
+from hustings.api.serializers import HustingSerializer
+from people.models import Person
 
 
 class PostcodeNotProvided(APIException):
@@ -148,15 +151,27 @@ def get(self, request):
         Returns the current timestamps used by the people and ballot recently
         updated importers
         """
+        data = {
+            "ballot_timestamp": None,
+            "person_timestamp": None,
+            "ballot_last_updated_url": None,
+            "person_last_updated_url": None,
+        }
+        api_base_url = f"{settings.YNR_BASE}/api/next/"
         try:
-            ballot_ts = PostElection.objects.last_updated_in_ynr().ynr_modified
-            ballot_ts = ballot_ts.isoformat()
+            ts = PostElection.objects.last_updated_in_ynr().ynr_modified
+            data["ballot_timestamp"] = ts.isoformat()
+            qs = urlencode({"last_updated": ts.isoformat()})
+            data["ballot_last_updated_url"] = f"{api_base_url}ballots/?{qs}"
         except PostElection.DoesNotExist:
-            ballot_ts = None
+            pass
 
-        return Response(
-            data={
-                "ballot_timestamp": ballot_ts,
-                "person_timestamp": Person.objects.latest().last_updated.isoformat(),
-            }
-        )
+        try:
+            ts = Person.objects.latest().last_updated.isoformat()
+            data["person_timestamp"] = ts
+            qs = urlencode({"last_updated": ts})
+            data["person_last_updated_url"] = f"{api_base_url}people/?{qs}"
+        except Person.DoesNotExist:
+            pass
+
+        return Response(data=data)

wcivf/apps/elections/management/commands/import_ballots.py

@@ -35,6 +35,13 @@ def add_arguments(self, parser):
             dest="recently_updated",
             help="Only import ballots updated since the last known update",
         )
+        parser.add_argument(
+            "--exclude-candidacies",
+            action="store_true",
+            dest="exclude_candidacies",
+            default=False,
+            help="Ignore candidacies when importing ballots",
+        )
 
     @time_function_length
     def populate_any_non_by_elections_field(self):
@@ -76,6 +83,7 @@ def handle(self, **options):
             force_metadata=options["force_metadata"],
             force_current_metadata=options["force_current_metadata"],
             recently_updated=options["recently_updated"],
+            exclude_candidacies=options["exclude_candidacies"],
         )
         importer.do_import()
         self.populate_any_non_by_elections_field()

wcivf/apps/people/management/commands/import_people.py

@@ -38,6 +38,13 @@ def add_arguments(self, parser):
             type=self.valid_date,
             help="Import changes since [datetime]",
         )
+        parser.add_argument(
+            "--exclude-candidacies",
+            action="store_true",
+            dest="exclude_candidacies",
+            default=False,
+            help="Ignore candidacies when importing people",
+        )
 
     def valid_date(self, value):
         return parse(value)
@@ -145,9 +152,10 @@ def add_people(self, results):
                         person_data=person,
                         person_obj=person_obj,
                     )
-                    self.update_candidacies(
-                        person_data=person, person_obj=person_obj
-                    )
+                    if not self.options["exclude_candidacies"]:
+                        self.update_candidacies(
+                            person_data=person, person_obj=person_obj
+                        )
                     # dont keep track of seen people in a recent update
                     continue
 
@@ -190,15 +198,23 @@ def update_candidacies(self, person_data, person_obj):
 
             # TODO check if the post/election could have changed and should be
             # used in defaults dict
+            defaults = {
+                "party_id": candidacy["party"]["legacy_slug"],
+                "list_position": candidacy["party_list_position"],
+                "elected": candidacy["elected"],
+                "party_name": candidacy["party_name"],
+                "party_description_text": candidacy["party_description_text"],
+            }
+            # TODO add this to YNR CandidacyOnPersonSerializer
+            if candidacy.get("result"):
+                num_ballots = candidacy["result"].get("num_ballots", None)
+                defaults["votes_cast"] = num_ballots
+
             obj, created = person_obj.personpost_set.update_or_create(
                 post_election=ballot,
                 post=ballot.post,
                 election=ballot.election,
-                defaults={
-                    "party_id": candidacy["party"]["legacy_slug"],
-                    "list_position": candidacy["party_list_position"],
-                    "elected": candidacy["elected"],
-                },
+                defaults=defaults,
             )
 
             msg = f"{obj} was {'created' if created else 'updated'}"

wcivf/apps/people/tests/test_import_people.py

@@ -19,6 +19,8 @@ def person_data(self):
                         "name": "Labour Party",
                         "legacy_slug": "party:53",
                     },
+                    "party_name": "Labour Party",
+                    "party_description_text": "Labour Party",
                     "ballot": {
                         "url": "http://candidates.democracyclub.org.uk/api/next/ballots/parl.romsey-and-southampton-north.2010-05-06/",
                         "ballot_paper_id": "parl.romsey-and-southampton-north.2010-05-06",
@@ -50,6 +52,8 @@ def test_update_candidacies(self, person_data, mocker):
                 "party_id": "party:53",
                 "list_position": 1,
                 "elected": False,
+                "party_name": "Labour Party",
+                "party_description_text": "Labour Party",
             },
         )