Skip to content

Commit

Permalink
Add ScrapedCandidate model methods, create RetentionContest objs for …
Browse files Browse the repository at this point in the history 
…'RECALL' elections
  • Loading branch information
@gordonje
gordonje committed Feb 7, 2017
1 parent 7a27c0b commit 762d67d
Show file tree
Hide file tree
Showing 3 changed files with 164 additions and 104 deletions.
95 changes: 8 additions & 87 deletions calaccess_processed/models/opencivicdata/elections/candidacy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,20 +10,10 @@
OCDIDField,
OCDBase,
)
from calaccess_processed.models.opencivicdata.division import Division
from calaccess_processed.models.opencivicdata.elections import ElectionIdentifier
from calaccess_processed.models.opencivicdata.elections.ballot_selection import CandidateSelection
from calaccess_processed.models.opencivicdata.elections.contest import CandidateContest
from calaccess_processed.models.opencivicdata.people_orgs import (
Organization,
Person,
Post,
)
from calaccess_processed.models.scraped import (
ScrapedCandidate,
CandidateScrapedElection,
)
import re
from calaccess_processed.models.scraped import ScrapedCandidate


class CandidacyManager(models.Manager):
Expand All @@ -41,70 +31,14 @@ def load_raw_data(self):
4. Person
5. Candidacy
"""
office_pattern = r'^(?P<type>[A-Z ]+)(?P<dist>\d{2})?$'

for sc in ScrapedCandidate.objects.all():
match = re.match(office_pattern, sc.office_name.upper())
office_type = match.groupdict()['type'].strip()
try:
district_num = int(match.groupdict()['dist'])
except TypeError:
pass
# get or create the post
post = sc.get_or_create_post()[0]

# prepare to get or create post
if office_type == 'STATE SENATE':
raw_post = {
'division': Division.objects.get(
subtype2='sldu',
subid2=str(district_num),
),
'organization': Organization.objects.get(
classification='upper',
),
'role': 'Senator',
}
elif office_type == 'ASSEMBLY':
raw_post = {
'division': Division.objects.get(
subtype2='sldl',
subid2=str(district_num),
),
'organization': Organization.objects.get(
classification='lower',
),
'role': 'Assembly Member',
}
else:
raw_post = {
'division': Division.objects.get(
id='ocd-division/country:us/state:ca'
),
'role': sc.office_name.title().replace('Of', 'of'),
}
if office_type == 'MEMBER BOARD OF EQUALIZATION':
raw_post['organization'] = Organization.objects.get(
name='State Board of Equalization',
)
elif office_type == 'SECRETARY OF STATE':
raw_post['organization'] = Organization.objects.get(
name='California Secretary of State',
)
else:
raw_post['organization'] = Organization.objects.get(
name='California State Executive Branch',
)

# get or create the Post
post = Post.objects.get_or_create(**raw_post)[0]

# get the scraped_election_id
scraped_election = CandidateScrapedElection.objects.get(
id=sc.election_id,
)
# get the election
elec = ElectionIdentifier.objects.get(
scheme='calaccess_id',
identifier=str(scraped_election.scraped_id),
identifier=sc.election.scraped_id,
).election

# get or create the CandidateContest (election and post)
Expand All @@ -122,32 +56,19 @@ def load_raw_data(self):
)
contest.posts.add(post)

if 'SPECIAL' in scraped_election.name:
if 'SPECIAL' in sc.election.name:
contest.is_unexpired_term = True
else:
contest.is_unexpired_term = False
# TODO: set runoff_for_contest, party and number_elected
contest.save()

# get or create the Person
person = Person.objects.get_using_filer_id(sc.scraped_id)
if not person:
# split and flip the original name string
split_name = sc.name.split(',')
split_name.reverse()
person = Person.objects.create(
sort_name=sc.name,
name=' '.join(split_name).strip()
)
person = sc.get_or_create_person()[0]

if sc.scraped_id != '':
person.identifiers.create(
scheme='calaccess_filer_id',
identifier=sc.scraped_id,
)
# check to see if the person has an candidacies for the election
# check to see if the person has any candidacies for the contest
q = Candidacy.objects.filter(
ballot_selection__contest__election=elec
ballot_selection__contest=contest
).filter(person=person)
if not q.exists():
# create the CandidateSelection
Expand Down
70 changes: 56 additions & 14 deletions calaccess_processed/models/opencivicdata/elections/contest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@
from __future__ import unicode_literals
from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from calaccess_processed.models.scraped import ScrapedProposition
from calaccess_processed.models.scraped import (
ScrapedCandidate,
ScrapedProposition,
)
from calaccess_processed.models.opencivicdata.elections import (
ElectionIdentifier,
)
Expand All @@ -16,6 +19,7 @@
OCDBase,
IdentifierBase,
)
from calaccess_processed.models.opencivicdata.people_orgs import Membership


@python_2_unicode_compatible
Expand Down Expand Up @@ -95,28 +99,66 @@ def load_raw_data(self):
division_obj = Division.objects.get(
id='ocd-division/country:us/state:ca'
)
if 'RECALL' in p.name:
if p.name == '2003 RECALL QUESTION':
# look up most recently scraped record for Gov. Gray Davis
scraped_candidate = ScrapedCandidate.objects.filter(
name='DAVIS, GRAY',
office_name='GOVERNOR',
).latest('created')
elif p.name == 'JUNE 3, 2008 - SPECIAL RECALL ELECTION - SENATE DISTRICT 12':
# look up most recently scraped record for Sen. Jeff Denham
scraped_candidate = ScrapedCandidate.objects.filter(
name='DENHAM, JEFF',
office_name='STATE SENATE 12',
).latest('created')
else:
# TODO: integrate previous election results
# look up the person currently in the post
raise Exception(
"Missing Membership (Person and Post) for %s." % p.name
)

# Measure is either an initiative or a referendum
ballot_measure_type = ''
if 'REFERENDUM' in p.name:
ballot_measure_type = 'referendum'
elif 'RECALL' in p.name:
ballot_measure_type = 'other'
# get or create person and post objects
person = scraped_candidate.get_or_create_person()[0]
post = scraped_candidate.get_or_create_post()[0]
# get or create membership object
membership = Membership.objects.get_or_create(
person=person,
post=post,
organization=post.organization,
)[0]
# create the retention contest
contest = RetentionContest.objects.create(
election=election_obj,
division=division_obj,
name=p.name,
ballot_measure_type='initiative',
membership=membership,
)
else:
ballot_measure_type = 'initiative'
if 'REFERENDUM' in p.name:
ballot_measure_type = 'referendum'
elif 'INITIATIVE' in p.name or 'INITATIVE' in p.name:
ballot_measure_type = 'initiative'
else:
ballot_measure_type = 'ballot measure'

contest = self.create(
election=election_obj,
division=division_obj,
name=p.name,
ballot_measure_type=ballot_measure_type
)
contest = self.create(
election=election_obj,
division=division_obj,
name=p.name,
ballot_measure_type=ballot_measure_type,
)

contest.identifiers.create(
scheme='ScrapedProposition.scraped_id',
identifier=p.scraped_id,
)

contest.ballot_selections.add(selection='Yes')
contest.ballot_selections.add(selection='No')

return


Expand Down
103 changes: 100 additions & 3 deletions calaccess_processed/models/scraped.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@
from __future__ import unicode_literals
from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from calaccess_processed.models.opencivicdata.division import Division
from calaccess_processed.models.opencivicdata.people_orgs import (
Organization,
Person,
Post,
)
import re


class BaseScrapedModel(models.Model):
Expand Down Expand Up @@ -53,9 +60,9 @@ class CandidateScrapedElection(BaseScrapedElection):
)
sort_index = models.IntegerField(
null=True,
help_text="The index value is used to preserve sorting of elections, \
since multiple elections may occur in a year. A greater sort index \
corresponds to a more recent election."
help_text="The index value is used to preserve sorting of elections, "
"since multiple elections may occur in a year. A greater sort "
"index corresponds to a more recent election."
)

def __str__(self):
Expand Down Expand Up @@ -86,6 +93,96 @@ class ScrapedCandidate(BaseScrapedModel):
def __str__(self):
return self.name

def get_or_create_post(self):
"""
Get or create a Post object using the ScrapedCandidate office_name value.
Returns a tuple (Post object, created), where created is a boolean
specifying whether a Post was created.
"""
office_pattern = r'^(?P<type>[A-Z ]+)(?P<dist>\d{2})?$'

match = re.match(office_pattern, self.office_name.upper())
office_type = match.groupdict()['type'].strip()
try:
district_num = int(match.groupdict()['dist'])
except TypeError:
pass

# prepare to get or create post
raw_post = {'label': self.office_name.title().replace('Of', 'of')}

if office_type == 'STATE SENATE':
raw_post['division'] = Division.objects.get(
subtype2='sldu',
subid2=str(district_num),
)
raw_post['organization'] = Organization.objects.get(
classification='upper',
)
raw_post['role'] = 'Senator'
elif office_type == 'ASSEMBLY':
raw_post['division'] = Division.objects.get(
subtype2='sldl',
subid2=str(district_num),
)
raw_post['organization'] = Organization.objects.get(
classification='lower',
)
raw_post['role'] = 'Assembly Member'
else:
raw_post['division'] = Division.objects.get(
id='ocd-division/country:us/state:ca'
)
if office_type == 'MEMBER BOARD OF EQUALIZATION':
raw_post['organization'] = Organization.objects.get(
name='State Board of Equalization',
)
raw_post['role'] = 'Board Member'
elif office_type == 'SECRETARY OF STATE':
raw_post['organization'] = Organization.objects.get(
name='California Secretary of State',
)
raw_post['role'] = raw_post['label']
else:
raw_post['organization'] = Organization.objects.get(
name='California State Executive Branch',
)
raw_post['role'] = raw_post['label']

return Post.objects.get_or_create(**raw_post)

def get_or_create_person(self):
"""
Get or create a Person object using the ScrapedCandidate name and scraped_id values.
If a Person object is created and scraped_id is not blank, a PersonIdentifier
object is also created.
Returns a tuple (Person object, created), where created is a boolean
specifying whether a Person was created.
"""
person = Person.objects.get_using_filer_id(self.scraped_id)

if not person:
# split and flip the original name string
split_name = self.name.split(',')
split_name.reverse()
person = Person.objects.create(
sort_name=self.name,
name=' '.join(split_name).strip()
)
if self.scraped_id != '':
person.identifiers.create(
scheme='calaccess_filer_id',
identifier=self.scraped_id,
)
created = True
else:
created = False

return (person, created)


@python_2_unicode_compatible
class PropositionScrapedElection(BaseScrapedElection):
Expand Down

0 comments on commit 762d67d

Please sign in to comment.