-
-
Notifications
You must be signed in to change notification settings - Fork 43
/
candidacies.py
410 lines (365 loc) · 14.2 KB
/
candidacies.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Proxy models for augmenting our source data tables with methods useful for processing.
"""
from __future__ import unicode_literals
from calaccess_processed.models.json_funcs import (
JSONArrayLength,
JSONExtractPath,
MaxFromJSONIntegerArray,
)
from django.db import models
from django.db.models import (
IntegerField,
Case,
Count,
F,
Max,
Q,
When,
)
from django.db.models.functions import Cast
from opencivicdata.core.models import Membership
from opencivicdata.elections.models import (
Candidacy,
CandidateContest,
CandidacySource,
)
from postgres_copy import CopyQuerySet
from .base import OCDProxyModelMixin
from .elections import OCDElectionProxy
from .people import OCDPersonProxy
class OCDCandidacyQuerySet(CopyQuerySet):
"""
Custom QuerySet for the OCD Candidacy model.
"""
def get_by_filer_id(self, filer_id):
"""
Returns a Candidacy object linked to a CALACCESS filer_id, if it exists.
"""
return self.get(
person__identifiers__scheme='calaccess_filer_id',
person__identifiers__identifier=filer_id,
)
def get_by_name(self, name):
"""
Returns a Candidacy object with the provided name from the CALACCESS database or scrape.
"""
return self.get(
Q(candidate_name=name) |
Q(person__name=name) |
Q(person__other_names__name=name)
)
class OCDCandidacyManager(models.Manager):
"""
Manager for custom methods on the OCDCandidacyProxy model.
"""
def get_queryset(self):
"""
Returns the custom QuerySet for this manager.
"""
return OCDCandidacyQuerySet(self.model, using=self._db)
def matched_form501_ids(self):
"""
Return all the Form 501 filing ids matched to a candidacy record.
"""
return [
i['extras']['form501_filing_ids'] for i in
self.get_queryset().filter(extras__has_key='form501_filing_ids').values('extras')
]
def get_or_create_from_calaccess(
self,
contest,
candidate_name_dict,
candidate_status="filed",
candidate_filer_id=None
):
"""
Get or create a Candidacy object with data from the CAL-ACCESS database.
First, try getting an existing Candidacy within the given CandidateContest
linked to a Person with the provided filer_id. If matched and the matched Person
has different current name and doesn't have the provided name as an other name,
add the other name.
Next, try getting an existing Candidacy within the given CandidateContest
linked to a Person with provided name (as default name or other name). If
matched and match candidate doesn't already have filer_id, add the filer_id.
If no match or if the matched person already has a different filer_id, create
a new Candidacy (this may also create a new Person record).
Returns a tuple (Candidacy object, created), where created is a boolean
specifying whether a Candidacy was created.
"""
candidacy = None
# first, try matching to existing candidate in contest with filer_id
if candidate_filer_id:
try:
candidacy = self.model.objects.filter(contest=contest).get_by_filer_id(candidate_filer_id)
except self.model.DoesNotExist:
pass
else:
candidacy_created = False
# if provided name not person's current name and not linked to person add it
candidacy.person.add_other_name(
candidate_name_dict['name'],
'Matched on CandidateContest and calaccess_filer_id'
)
# if filer_id match fails (or no filer_id), try matching to candidate
# in contest with provided name
if not candidacy:
try:
candidacy = self.model.objects.filter(contest=contest).get_by_name(candidate_name_dict['name'])
except self.model.MultipleObjectsReturned:
# weird case when someone filed for the same race
# with three different filer_ids
if candidate_name_dict['sort_name'] == 'MC NEA, DOUGLAS A.':
candidacy = None
except self.model.DoesNotExist:
pass
else:
candidacy_created = False
# if filer_id provided
if candidate_filer_id:
# check to make sure candidate with same name doesn't have diff filer_id
if candidacy.person.identifiers.filter(scheme='calaccess_filer_id').exists():
# if so, don't conflate
candidacy = None
else:
# if so, add filer_id to existing candidate
person = candidacy.person
person.refresh_from_db()
person.__class__ = OCDPersonProxy
person.add_filer_id(candidate_filer_id)
# if no matched candidate yet, make a new one
if not candidacy:
# First make a Person object
person, person_created = OCDPersonProxy.objects.get_or_create_from_calaccess(
candidate_name_dict,
candidate_filer_id=candidate_filer_id
)
person.add_other_name(candidate_name_dict['name'], 'From {} candidacy'.format(contest))
# Then make the Candidacy
candidacy = OCDCandidacyProxy.objects.create(
contest=contest,
person=person,
post=contest.posts.all()[0].post,
candidate_name=candidate_name_dict['name'],
registration_status=candidate_status,
)
candidacy_created = True
# if provided registration does not equal the default, update
if candidate_status != 'filed' and candidate_status != candidacy.registration_status:
candidacy.registration_status = candidate_status
candidacy.save()
# make sure Person name is same as most recent candidate_name
person = candidacy.person
person.refresh_from_db()
person.__class__ = OCDPersonProxy
person.update_name()
# Pass it back out.
return candidacy, candidacy_created
class OCDCandidacyProxy(Candidacy, OCDProxyModelMixin):
"""
A proxy on the OCD Candidacy model with helper methods.
"""
objects = OCDCandidacyManager.from_queryset(CopyQuerySet)()
copy_to_fields = (
('id',),
('candidate_name',),
('person',),
('party',),
('contest',),
('post',),
('is_incumbent',),
('registration_status',),
('top_ticket_candidacy',),
('filed_date',),
('created_at',),
('updated_at',),
('extras',),
('locked_fields',),
)
class Meta:
"""
Make this a proxy model.
"""
proxy = True
@property
def election_proxy(self):
"""
Returns the proxied OCDElectionProxy linked to this candidacy.
"""
return OCDElectionProxy.objects.get(id=self.contest.election_id)
def link_form501(self, form501_id):
"""
Link an id of a Form501Filing to a Candidacy, if it isn't already.
"""
# Check if the attribute is already there
if 'form501_filing_ids' in self.extras:
# If it is, check if we already have this id
if form501_id not in self.extras['form501_filing_ids']:
# If we don't, append it to the list
self.extras['form501_filing_ids'].append(form501_id)
# Save out
self.save()
# If the attribute isn't there, go ahead and add it.
else:
self.extras['form501_filing_ids'] = [form501_id]
# Save out
self.save()
def update_from_form501(self):
"""
Set Candidacy fields using data extracted from linked Form501Filings.
"""
from calaccess_processed.models import Form501Filing
# get all Form501Filing linked to Candidacy
filing_ids = self.extras['form501_filing_ids']
filings = Form501Filing.objects.filter(filing_id__in=filing_ids)
# keep the earliest filed_date
first_filed_date = filings.earliest('date_filed').date_filed
# If the filed dates don't match, update them
if self.filed_date != first_filed_date:
self.filed_date = first_filed_date
self.save()
# set registration status to "withdrawn" based on statement_type of latest Form501
latest = filings.latest('date_filed')
if latest.statement_type == '10003': # <-- This is the code for withdrawn
# If the candidacy hasn't been marked that way, update it now
if self.registration_status != 'withdrawn':
self.registration_status = 'withdrawn'
self.save()
# set party based on latest Form501 where the field is populated
latest_party = filings.filter(
party__isnull=False
).latest('date_filed').get_party()
if latest_party != self.party:
self.party = latest_party
self.save()
def check_incumbency(self):
"""
Check if the Candidacy is for the incumbent officeholder.
Return True if:
* Membership exists for the Person and Post linked to the Candidacy, and
* Membership.end_date is NULL or has a year later than Election.date.year.
"""
incumbent_q = Membership.objects.filter(
post=self.post,
person=self.person,
).annotate(
# Cast end_date's value as an int, treat '' as NULL
end_year=Cast(
Case(When(end_date='', then=None)),
IntegerField(),
)
).filter(
Q(end_year__gt=self.election.date.year) |
Q(end_date='')
)
if incumbent_q.exists():
return True
else:
return False
@property
def filer_ids(self):
"""
Returns the CAL-ACCESS filer_id linked with the object, if any.
"""
return self.person.identifiers.filter(scheme="calaccess_filer_id")
@property
def form501_filing_ids(self):
"""
Returns any linked Form 501 filing ids.
"""
try:
return self.extras['form501_filing_ids']
except KeyError:
return []
@property
def form501s(self):
"""
Returns any linked Form 501 objects.
"""
from calaccess_processed.models import Form501Filing
return Form501Filing.objects.filter(filing_id__in=self.form501_filing_ids)
class OCDCandidacySourceProxy(CandidacySource, OCDProxyModelMixin):
"""
A proxy on the OCD CandidacySource model.
"""
objects = CopyQuerySet.as_manager()
class Meta:
"""
Make this a proxy model.
"""
proxy = True
class OCDFlatCandidacyManager(models.Manager):
"""
Custom manager for flattening the contents of the OCD Candidacy model.
"""
def get_queryset(self):
"""
Returns the custom QuerySet for this manager.
"""
return super(
OCDFlatCandidacyManager, self
).get_queryset().filter(
Q(person__identifiers__scheme='calaccess_filer_id') |
Q(person__identifiers__isnull=True)
).annotate(
name=F('candidate_name'),
office=F('post__label'),
party_name=F('party__name'),
election_name=F('contest__election__name'),
election_date=F('contest__election__date'),
special_election=F('contest__previous_term_unexpired'),
ocd_person_id=F('person__id'),
ocd_candidacy_id=F('id'),
ocd_election_id=F('contest__election'),
ocd_post_id=F('post__id'),
ocd_contest_id=F('contest'),
ocd_party_id=F('party'),
latest_calaccess_filer_id=Max('person__identifiers__identifier'),
calaccess_filer_id_count=Count('person__identifiers__identifier'),
latest_form501_filing_id=MaxFromJSONIntegerArray(
'extras', 'form501_filing_ids'
),
form501_filing_count=JSONArrayLength(
JSONExtractPath('extras', 'form501_filing_ids')
),
)
class OCDFlatCandidacyProxy(Candidacy, OCDProxyModelMixin):
"""
Every candidate for a public office recorded in CAL-ACCESS.
"""
objects = OCDFlatCandidacyManager.from_queryset(CopyQuerySet)()
copy_to_fields = (
('name',),
('party_name',
'Name of the political party that nominated the candidate or would '
'nominate the candidate (as in the case of a partisan primary).',),
('election_name',),
('election_date',),
('office',
'Public office for which the candidate is seeking election.',),
('is_incumbent',),
('special_election', CandidateContest._meta.get_field('previous_term_unexpired').help_text),
('created_at',),
('updated_at',),
('ocd_person_id', Candidacy._meta.get_field('person').help_text),
('ocd_candidacy_id',),
('ocd_election_id', CandidateContest._meta.get_field('election').help_text),
('ocd_post_id', Candidacy._meta.get_field('post').help_text),
('ocd_contest_id', Candidacy._meta.get_field('contest').help_text),
('ocd_party_id', Candidacy._meta.get_field('party').help_text),
('latest_calaccess_filer_id',
'Most recent filer_id assigned to the person in CAL-ACCESS.',),
('calaccess_filer_id_count',
'Count of filer_ids assigned to the person in CAL-ACCESS.',),
('latest_form501_filing_id', "CAL-ACCESS identifier for the candidate's "
"most recent Form 501 filing."),
('form501_filing_count', 'Count of Form 501s filed by the candidate.'),
)
class Meta:
"""
Make this a proxy model.
"""
proxy = True
verbose_name_plural = 'candidates'