Permalink
Browse files

Merge branch 'master' of github.com:sunlightlabs/datacommons

  • Loading branch information...
2 parents b630136 + 09c11b2 commit dfebf64f5b8590b2bac8574bbea508e151940509 @arowla arowla committed Apr 27, 2012
@@ -34,7 +34,9 @@
coalesce(rs.document_count, 0)::integer,
coalesce(f.member_count, 0)::integer,
coalesce(f.committee_count, 0)::integer,
- coalesce(indexp.spending_amount, 0)::float
+ coalesce(indexp.spending_amount, 0)::float,
+ coalesce(fec_committee.total_raised, 0)::float,
+ coalesce(fec_committee.count, 0)::integer
from
(select *
from agg_entities
@@ -84,11 +86,16 @@
from agg_fec_indexp_totals
where entity_id = %s) indexp
using (cycle)
+ full outer join (
+ select cycle, total_raised, count
+ from agg_fec_committee_summaries
+ where entity_id = %s) fec_committee
+ using (cycle)
"""
def get_totals(entity_id):
totals = dict()
- for row in execute_top(get_totals_stmt, *[entity_id] * 10):
+ for row in execute_top(get_totals_stmt, *[entity_id] * 11):
totals[row[0]] = dict(zip(EntityHandler.totals_fields, row[1:]))
return totals
@@ -104,7 +111,7 @@ class EntityHandler(BaseHandler):
'epa_actions_count',
'regs_docket_count', 'regs_document_count', 'regs_submitted_docket_count', 'regs_submitted_document_count',
'faca_member_count', 'faca_committee_count',
- 'independent_expenditure_amount']
+ 'independent_expenditure_amount', 'fec_total_raised', 'fec_summary_count']
ext_id_fields = ['namespace', 'id']
def read(self, request, entity_id):
@@ -160,7 +167,7 @@ class EntitySearchHandler(BaseHandler):
'id', 'name', 'type',
'count_given', 'count_received', 'count_lobbied',
'total_given','total_received', 'firm_income', 'non_firm_spending',
- 'state', 'party', 'seat', 'lobbying_firm'
+ 'state', 'party', 'seat', 'lobbying_firm', 'is_superpac'
]
stmt = """
@@ -173,7 +180,7 @@ class EntitySearchHandler(BaseHandler):
coalesce(a.recipient_amount, 0)::float,
coalesce(l.firm_income, 0)::float,
coalesce(l.non_firm_spending, 0)::float,
- pm.state, pm.party, pm.seat, om.lobbying_firm
+ pm.state, pm.party, pm.seat, om.lobbying_firm, om.is_superpac
from matchbox_entity e
inner join (select distinct entity_id
from matchbox_entityalias ea
View
@@ -206,6 +206,32 @@ where
create index fec_committee_itemized_committee_id on fec_committee_itemized (committee_id);
+drop table if exists agg_fec_committee_summaries;
+create table agg_fec_committee_summaries as
+select
+ cycle, entity_id,
+ sum(total_receipts - (loan_repayments + refunds_to_individuals + refunds_to_committees)) as total_raised,
+ sum(individual_contributions - refunds_to_individuals) as individual_contributions,
+ sum(contributions_from_other_committees) as contributions_from_other_committees,
+ sum(transfers_from_affiliates) as transfers_from_affiliates,
+ sum(nonfederal_transfers_received) as nonfederal_transfers_received,
+ sum(total_loans_received) as total_loans_received,
+ sum(total_disbursements) as total_disbursements,
+ sum(cash_close_of_period) as cash_close_of_period,
+ sum(debts_owed) as debts_owed,
+ sum(contributions_to_committees) as contributions_to_committees,
+ sum(independent_expenditures_made) as independent_expenditures_made,
+ sum(party_coordinated_expenditures_made) as party_coordinated_expenditures_made,
+ sum(nonfederal_expenditure_share) as nonfederal_expenditure_share,
+ min(through_date) as min_through_date,
+ max(through_date) as max_through_date,
+ count(*)
+from fec_committee_summaries c
+inner join matchbox_entityattribute a on c.committee_id = a.value and a.namespace = 'urn:fec:committee'
+cross join (values (-1), (2012)) as cycles (cycle)
+group by cycle, entity_id;
+
+
-- these three are unfortunately not true in the data
-- alter table fec_candidates add constraint fec_candidates_committee_id foreign key (committee_id) references fec_committees (committee_id);
@@ -0,0 +1,32 @@
+drop view organization_metadata_latest_cycle_view;
+create or replace view organization_metadata_latest_cycle_view as
+ select distinct on (entity_id)
+ entity_id,
+ cycle,
+ lobbying_firm,
+ parent_entity_id,
+ industry_entity_id,
+ subindustry_entity_id,
+ is_superpac
+ from matchbox_organizationmetadata
+ where cycle <= 2012
+ order by entity_id, cycle desc
+;
+
+create or replace view politician_metadata_latest_cycle_view as
+ select distinct on (entity_id)
+ entity_id,
+ cycle,
+ state,
+ state_held,
+ district,
+ district_held,
+ party,
+ seat,
+ seat_held,
+ seat_status,
+ seat_result
+ from matchbox_politicianmetadata
+ order by entity_id, cycle desc
+;
+
@@ -0,0 +1,106 @@
+
+
+create table tmp_attributes_to_delete as
+select value
+from matchbox_entityattribute a
+inner join fec_committees c1 on committee_id = value
+where
+ namespace = 'urn:fec:committee'
+ and committee_type = 'O'
+ and exists (
+ select *
+ from matchbox_entityattribute b
+ inner join fec_committees c2 on c2.committee_id = b.value
+ where
+ a.entity_id = b.entity_id
+ and b.namespace = 'urn:fec:committee'
+ and a.value != b.value
+ and c2.committee_type != 'O'
+ );
+
+begin;
+
+delete from matchbox_entityattribute where value in (table tmp_attributes_to_delete);
+
+
+drop table if exists tmp_entities_to_create;
+create table tmp_entities_to_create as
+select committee_name, committee_id
+from fec_committee_summaries
+where
+ committee_type = 'O'
+ and not exists (
+ select *
+ from matchbox_entityattribute
+ where
+ namespace = 'urn:fec:committee'
+ and value = committee_id
+ );
+
+\copy tmp_entities_to_create to new_entities.csv csv
+
+
+-- # create entities with the following script
+-- import csv
+-- from dcentity.entity import build_entity
+-- from django.db import transaction
+--
+-- with transaction.commit_on_success():
+-- for (name, id) in csv.reader(open('new_entities.csv', 'r')):
+-- build_entity(name, 'organization', [('urn:fec:committee', id)])
+
+
+rollback;
+
+
+
+alter table matchbox_organizationmetadata add column is_superpac boolean not null default false;
+
+update matchbox_organizationmetadata
+set is_superpac = true
+from matchbox_entityattribute a
+inner join fec_committees on committee_id = value
+where
+ matchbox_organizationmetadata.entity_id = a.entity_id
+ and matchbox_organizationmetadata.cycle = 2012
+ and namespace = 'urn:fec:committee'
+ and committee_type = 'O';
+
+
+insert into matchbox_organizationmetadata (entity_id, cycle, is_superpac)
+select entity_id, 2012, true
+from matchbox_entityattribute a
+inner join fec_committees on committee_id = value
+where
+ namespace = 'urn:fec:committee'
+ and committee_type = 'O'
+ and not exists (
+ select *
+ from matchbox_organizationmetadata existing
+ where
+ existing.entity_id = a.entity_id
+ and existing.cycle = 2012
+ );
+
+
+
+-- add the org-> SuperPAC links
+
+-- first, load tmp_crp_committees, as done in crp_load_fec_ids command
+
+insert into matchbox_superpaclinks (superpac_entity_id, controlling_org_entity_id)
+select a.entity_id, l.entity_id
+from tmp_crp_committees c
+inner join matchbox_entityattribute a on a.namespace = 'urn:fec:committee' and c.CmteID = a.value
+inner join fec_committees f on f.committee_id = a.value
+inner join matchbox_entityalias l on lower(alias) = lower(c.ultorg)
+where
+ a.entity_id != l.entity_id
+ and f.committee_type = 'O'
+ and not exists (
+ select *
+ from matchbox_superpaclinks existing
+ where
+ existing.superpac_entity_id = a.entity_id
+ and existing.controlling_org_entity_id = l.entity_id
+ );
View
@@ -111,25 +111,31 @@ def _get_all_metadata_as_dict(self):
metadata['federal_politician_entities'] = [ x.federal_politician_entity for x in self.federal_offices_held.all() ]
- elif self.type == 'organization' and hasattr(self, 'organization_metadata_by_cycle'):
- for data_by_cycle in self.organization_metadata_by_cycle.all():
- metadata[data_by_cycle.cycle] = data_by_cycle.to_dict()
- del(metadata[data_by_cycle.cycle]['cycle'])
- del(metadata[data_by_cycle.cycle]['id'])
- del(metadata[data_by_cycle.cycle]['entity'])
-
- # assign latest cycle to old fields for backwards compatibility
- # (might not need this going forward)
- if hasattr(self, 'organization_metadata_for_latest_cycle'):
- latest = self.organization_metadata_for_latest_cycle.to_dict()
- del(latest['cycle'])
- del(latest['entity'])
- metadata.update(latest)
- else:
- metadata['parent_entity'] = ''
- metadata['industry_entity'] = ''
- metadata['lobbying_firm'] = False
- metadata['child_entities'] = []
+ elif self.type == 'organization':
+ if hasattr(self, 'organization_metadata_by_cycle'):
+ for data_by_cycle in self.organization_metadata_by_cycle.all():
+ metadata[data_by_cycle.cycle] = data_by_cycle.to_dict()
+ del(metadata[data_by_cycle.cycle]['cycle'])
+ del(metadata[data_by_cycle.cycle]['id'])
+ del(metadata[data_by_cycle.cycle]['entity'])
+
+ # assign latest cycle to old fields for backwards compatibility
+ # (might not need this going forward)
+ if hasattr(self, 'organization_metadata_for_latest_cycle'):
+ latest = self.organization_metadata_for_latest_cycle.to_dict()
+ del(latest['cycle'])
+ del(latest['entity'])
+ metadata.update(latest)
+ else:
+ metadata['parent_entity'] = ''
+ metadata['industry_entity'] = ''
+ metadata['lobbying_firm'] = False
+ metadata['child_entities'] = []
+
+ if hasattr(self, 'controlling_org'):
+ metadata['controlling_org'] = self.controlling_org.controlling_org_entity.public_representation()
+ if self.affiliated_superpacs.count() > 0:
+ metadata['affiliated_superpacs'] = [ x.superpac_entity.public_representation() for x in self.affiliated_superpacs.all() ]
elif self.type == 'individual':
# in the future, individuals should probably have their own metadata table,
@@ -221,6 +227,8 @@ class OrganizationMetadata(ExtensibleModel):
parent_entity = models.ForeignKey(Entity, related_name='child_entity_set_for_cycle', null=True, db_index=True)
industry_entity = models.ForeignKey(Entity, related_name='industry_entity_for_cycle', null=True, db_index=True)
subindustry_entity = models.ForeignKey(Entity, related_name='subindustry_entity_for_cycle', null=True, db_index=True)
+ is_superpac = models.BooleanField(default=False)
+
@property
def child_entities(self):
@@ -241,6 +249,7 @@ class OrganizationMetadataLatest(ExtensibleModel):
lobbying_firm = models.BooleanField(default=False)
parent_entity = models.ForeignKey(Entity, related_name='child_entity_set', null=True)
industry_entity = models.ForeignKey(Entity, related_name='industry_entity', null=True)
+ is_superpac = models.BooleanField(default=False)
@property
def child_entities(self):
@@ -330,6 +339,14 @@ class Meta:
db_table = 'matchbox_statefederal'
+class SuperPACLinks(models.Model):
+ superpac_entity = models.OneToOneField(Entity, related_name='controlling_org')
+ controlling_org_entity = models.ForeignKey(Entity, related_name='affiliated_superpacs')
+
+ class Meta:
+ db_table = 'matchbox_superpaclinks'
+
+
class VotesmartInfo(models.Model):
entity = models.OneToOneField(Entity, related_name='votesmart_info', null=False)

0 comments on commit dfebf64

Please sign in to comment.