Skip to content

Commit

Permalink
Close #44
Browse files Browse the repository at this point in the history
  • Loading branch information
@gordonje
gordonje committed Feb 3, 2017
1 parent f5993bf commit e4a9afa
Show file tree
Hide file tree
Showing 4 changed files with 116 additions and 98 deletions.
100 changes: 57 additions & 43 deletions calaccess_processed/models/opencivicdata/elections/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,11 @@ def load_raw_data(self):
"""
Load Election model from CandidateScrapedElection and PropositionScrapedElection.
"""
self.all().delete()

prop_name_pattern = r'^(?P<date>^[A-Z]+\s\d{1,2},\s\d{4})\s(?P<name>.+)$'

# start by loading the prop elections, which include a date in the name
for p in PropositionScrapedElection.objects.all():
# extract the name and date
match = re.match(prop_name_pattern, p.name)
dt_obj = timezone.make_aware(
timezone.datetime.strptime(
Expand All @@ -60,27 +59,33 @@ def load_raw_data(self):
dt_obj.year,
match.groupdict()['name'],
).upper()

# try getting an existing OCD election with the same date
try:
elex = self.get(start_time=dt_obj)
elec = self.get(start_time=dt_obj)
except self.model.DoesNotExist:
elex = self.create(start_time=dt_obj, name=name)
# or make a new one
elec = self.create(start_time=dt_obj, name=name)
# TODO: Set adminstrative_org, source, etc.
else:
# if election already exists and is named 'SPECIAL' or 'RECALL'
if 'SPECIAL' in elex.name.upper() or 'RECALL' in elex.name.upper():
if (
'SPECIAL' in elec.name.upper() or
'RECALL' in elec.name.upper()
):
# and the matched election's name includes either 'GENERAL'
# or 'PRIMARY'...
if (
re.match(r'^\d{4} GENERAL$', name) or
re.match(r'^\d{4} PRIMARY$', name)
):
# update the name
elex.name = name
elex.save()
elec.name = name
elec.save()

# whether creating or matching, add the id
elex.identifiers.create(
# whether creating or matching, attempt to create the new id
# TODO: these prop ids will get flushed with each scraped and might change...
# Keep them all anyway?
elec.identifiers.get_or_create(
scheme='PropositionScrapedElection.id',
identifier=str(p.id)
)
Expand Down Expand Up @@ -154,41 +159,49 @@ def load_raw_data(self):

# then loop over the candidate elections
for c in CandidateScrapedElection.objects.all():
# if the name is in the list of special elections
if c.name in (x[0] for x in cand_elections_w_dates):
date = dict(cand_elections_w_dates)[c.name]
dt_obj = timezone.make_aware(
timezone.datetime.strptime(
date,
'%Y-%m-%d',
)
)
# get or create the special election
try:
elex = self.get(start_time=dt_obj)
except self.model.DoesNotExist:
elex = self.create(
start_time=dt_obj,
name=c.name,
is_statewide=False,
)
else:
# assume the candidate election name is in the '{year} {type}' format
try:
elex = self.get(name=c.name)
except Election.DoesNotExist:
# this recall election occurred on the same date as 2008 primary
# http://www.sos.ca.gov/elections/prior-elections/special-elections/special-recall-election-senate-district-12-june-3-2008/
# http://www.sos.ca.gov/elections/prior-elections/statewide-election-results/statewide-direct-primary-election-june-3-2008/
if c.name == '2008 RECALL (STATE SENATE 12)':
elex = self.get(name='2008 PRIMARY')
else:
raise Exception('Missing record for %s' % c.name)

elex.identifiers.create(
# skip if the candidata election id is already linked to an OCD election
if ElectionIdentifier.objects.filter(
scheme='calaccess_id',
identifier=str(c.scraped_id)
)
).exists():
pass
else:
# if the name is in the list of special elections
if c.name in (x[0] for x in cand_elections_w_dates):
date = dict(cand_elections_w_dates)[c.name]
dt_obj = timezone.make_aware(
timezone.datetime.strptime(
date,
'%Y-%m-%d',
)
)
# get or create the special election
try:
elec = self.get(start_time=dt_obj)
except self.model.DoesNotExist:
elec = self.create(
start_time=dt_obj,
name=c.name,
is_statewide=False,
)
else:
# assume the candidate election name is in the
# '{year} {type}' format
try:
elec = self.get(name=c.name)
except Election.DoesNotExist:
# this recall election occurred on the same date as 2008 primary
# http://www.sos.ca.gov/elections/prior-elections/special-elections/special-recall-election-senate-district-12-june-3-2008/
# http://www.sos.ca.gov/elections/prior-elections/statewide-election-results/statewide-direct-primary-election-june-3-2008/
if c.name == '2008 RECALL (STATE SENATE 12)':
elec = self.get(name='2008 PRIMARY')
else:
raise Exception('Missing record for %s' % c.name)

elec.identifiers.create(
scheme='calaccess_id',
identifier=str(c.scraped_id)
)
# Remove office from name of special elections with multiple races
special_elections = self.filter(
name__regex=r'^\d{4}\sSPECIAL\s.+\(.+$'
Expand Down Expand Up @@ -233,6 +246,7 @@ def __str__(self):
return self.name


@python_2_unicode_compatible
class ElectionIdentifier(IdentifierBase):
"""
Model for storing an OCD Election's other identifiers.
Expand Down
43 changes: 20 additions & 23 deletions calaccess_processed/models/opencivicdata/elections/candidacy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,6 @@ def load_raw_data(self):
4. Person
5. Candidacy
"""
# flush all the models to be loaded
Post.objects.all().delete()
CandidateContest.objects.all().delete()
CandidateSelection.objects.all().delete()
Person.objects.all().delete()
Candidacy.objects.all().delete()

office_pattern = r'^(?P<type>[A-Z ]+)(?P<dist>\d{2})?$'

for sc in ScrapedCandidate.objects.all():
Expand Down Expand Up @@ -132,12 +125,6 @@ def load_raw_data(self):
# TODO: set runoff_for_contest, party and number_elected
contest.save()

# create the CandidateSelection
selection = CandidateSelection.objects.create(
contest=contest,
# TODO: set endorsement_parties and is_write_in
)

# get or create the Person
person = Person.objects.get_using_filer_id(sc.scraped_id)
if not person:
Expand All @@ -154,16 +141,26 @@ def load_raw_data(self):
scheme='calaccess_filer_id',
identifier=sc.scraped_id,
)

# create the Candidacy
Candidacy.objects.create(
person=person,
ballot_name=sc.name,
post=post,
is_top_ticket=False,
candidate_selection=selection,
# TODO: set committee, is_incumbent and party
)
# check to see if the person has an candidacies for the election
q = Candidacy.objects.filter(
candidate_selection__contest__election=elec
).filter(person=person)
if not q.exists():
# create the CandidateSelection
selection = CandidateSelection.objects.create(
contest=contest,
)
# TODO: set or update the endorsement_parties and is_write_in

# create the Candidacy
Candidacy.objects.create(
person=person,
ballot_name=sc.name,
post=post,
is_top_ticket=False,
candidate_selection=selection,
# TODO: set committee, is_incumbent and party
)

return

Expand Down
55 changes: 32 additions & 23 deletions calaccess_processed/models/opencivicdata/elections/contest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,40 +56,49 @@ def load_raw_data(self):
"""
Load BallotMeasureContest model from ScrapedProposition.
"""
self.all().delete()

for p in ScrapedProposition.objects.all():
# Get the election
election_obj = ElectionIdentifier.objects.get(
scheme='PropositionScrapedElection.id',
identifier=str(p.election_id),
).election

# Get the division: CA statewide
division_obj = Division.objects.get(
id='ocd-division/country:us/state:ca'
q = BallotMeasureContestIdentifier.objects.filter(
scheme='calaccess_filer_id',
identifier=p.scraped_id,
)

# Measure is either an initiative or a referendum
ballot_measure_type = ''
if 'REFERENDUM' in p.name:
ballot_measure_type = 'r'
elif 'RECALL' in p.name:
ballot_measure_type = 'o'
# check if we already have a BallotMeasureContest for the prop
if q.exists():
# if so, make sure the name is up-to-date
if q[0].name != p.name:
q[0].name = p.name
q[0].save()
else:
ballot_measure_type = 'i'
# Get the division: CA statewide
division_obj = Division.objects.get(
id='ocd-division/country:us/state:ca'
)

contest = self.create(
election=election_obj,
division=division_obj,
name=p.name,
ballot_measure_type=ballot_measure_type
)
# Measure is either an initiative or a referendum
ballot_measure_type = ''
if 'REFERENDUM' in p.name:
ballot_measure_type = 'r'
elif 'RECALL' in p.name:
ballot_measure_type = 'o'
else:
ballot_measure_type = 'i'

contest.identifiers.create(
scheme='calaccess_filer_id',
identifier=p.scraped_id,
)
contest = self.create(
election=election_obj,
division=division_obj,
name=p.name,
ballot_measure_type=ballot_measure_type
)

contest.identifiers.create(
scheme='calaccess_filer_id',
identifier=p.scraped_id,
)

return

Expand Down
16 changes: 7 additions & 9 deletions calaccess_processed/models/opencivicdata/people_orgs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,27 +64,25 @@ def load_raw_data(self):
"""
Insert records for California state government organizations.
"""
self.all().delete()

exec_branch = self.create(
exec_branch = self.get_or_create(
name='California State Executive Branch',
classification='executive',
)
self.create(
)[0]
self.get_or_create(
name='State Board of Equalization',
classification='executive',
parent=exec_branch,
)
leg = self.create(
leg = self.get_or_create(
name='California State Legislature',
classification='legislature',
)
self.create(
)[0]
self.get_or_create(
name='California State Senate',
classification='upper',
parent=leg,
)
self.create(
self.get_or_create(
name='California State Assembly',
classification='lower',
parent=leg,
Expand Down

0 comments on commit e4a9afa

Please sign in to comment.