@@ -81,7 +81,7 @@ def query_census(location, hr_keys): # Use DISTINCT since some hr_keys map to multiple internal_keys (but should # have same value). #q = db.select('census', what='SUM(DISTINCT(value))', where=web.sqlors('hr_key=', hr_keys)+' AND location='+web.sqlquote(location)) - q = db.query('SELECT SUM(value) FROM (SELECT DISTINCT value, hr_key FROM census WHERE '+web.sqlors('hr_key=', hr_keys)+' AND location='+web.sqlquote(location)+') AS foo;') + q = db.query('SELECT SUM(value) FROM (SELECT DISTINCT value, hr_key FROM census WHERE '+web.sqlors('hr_key=', hr_keys)+' AND district_id='+web.sqlquote(location)+') AS foo;') if not q: return None return q[0].sum @@ -138,7 +138,7 @@ def get_total_pop(location): if __name__ == "__main__": from pprint import pprint - states = ['AK', 'AL', 'AR', 'AZ', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'GA', + states = ['AK', 'AL', 'AR', 'AZ', 'CA', 'CO', 'CT', 'DE', 'FL', 'GA', 'HI', 'IA', 'ID', 'IL', 'IN', 'KS', 'KY', 'LA', 'MA', 'MD', 'ME', 'MI', 'MN', 'MO', 'MS', 'MT', 'NC', 'ND', 'NE', 'NH', 'NJ', 'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD', 'TN', 'TX', 'UT', 'VA', 'VT', census.py @@ -43,7 +43,7 @@ NJS=$(PARSE)/almanac.njs \ all: database clean: - rm $(JSONS) .schema .json .bills .interests .votesmart .photos .fec .census .soi + -rm $(JSONS) .schema .json .bills .interests .votesmart .photos .fec .census .soi cleanparse: rm $(NJS) @@ -121,14 +121,22 @@ $(LOAD)/bill.tsv $(LOAD)/vote.tsv: load/bills.py $(PYTHON) load/fec.py touch $@ -.zip4: $(LOAD)/zip4.tsv - $(DBCLIENT) $(WATCHDOG_TABLE) < $(LOAD)/zip4.tsv +.zip4: $(LOAD)/zip4.tsv .json + echo "COPY zip4 (zip,plus4,district_id) FROM '${PWD}/../data/load/zip4.tsv';" | $(DBCLIENT) $(WATCHDOG_TABLE) touch $@ .soi: .schema .json .zip4 $(PYTHON) load/soi.py touch $@ +$(LOAD)/census_meta.tsv $(LOAD)/census_data.tsv: load/census.py parse/census.py + python load/census.py + +.census: $(LOAD)/census_meta.tsv $(LOAD)/census_data.tsv .schema .json + $(DBCLIENT) $(WATCHDOG_TABLE) < $(LOAD)/census_meta.tsv + $(DBCLIENT) $(WATCHDOG_TABLE) < $(LOAD)/census_data.tsv + touch $@ + # new way of doing things: parse to .njs files parse: $(NJS) @@ -156,12 +164,3 @@ $(PARSE)/soi.njs: parse/soi.py $(CRAWL)/irs/soi/ $(PARSE)/voteview.njs: parse/voteview.py $(PYTHON) parse/voteview.py > $@.tmp mv $@.tmp $@ - -$(LOAD)/census_meta.tsv $(LOAD)/census_data.tsv: load/census.py parse/census.py - python load/census.py - -.census: $(LOAD)/census_meta.tsv $(LOAD)/census_data.tsv .schema - $(DBCLIENT) $(WATCHDOG_TABLE) < $(LOAD)/census_meta.tsv - $(DBCLIENT) $(WATCHDOG_TABLE) < $(LOAD)/census_data.tsv - touch $@ - import/Makefile @@ -95,11 +95,11 @@ def loadbill(fn, maplightid=None, batch_mode=False): def main(): from bulk_loader import bulk_loader_db for c,fn in enumerate(glob.glob(GOVTRACK_CRAWL+'/us/*/bills/*.xml')): - loadbill(fn,batch_mode=True) + loadbill(fn, batch_mode=True) db = bulk_loader_db(os.environ.get('WATCHDOG_TABLE', 'watchdog_dev')) - bill_cols = ['id', 'session', 'type', 'number', 'introduced', 'title', 'sponsor', 'summary', 'maplightid', 'yeas', 'neas'] + bill_cols = ['id', 'session', 'type', 'number', 'introduced', 'title', 'sponsor_id', 'summary', 'maplightid', 'yeas', 'neas'] db.open_table('bill', bill_cols, delete_first=True, filename=DATA_DIR+'load/bill.tsv') vote_col = ['bill_id', 'politician_id', 'vote'] db.open_table('vote', vote_col, delete_first=True, filename=DATA_DIR+'load/vote.tsv') import/load/bills.py @@ -47,8 +47,11 @@ def load_census_data(type): loc_code = '%s-%02d' % (geo[logrecno]['STATE'], int(geo[logrecno]['CD110'])) else: continue + # Fix some districts: + if loc_code in ['DC','PR']: loc_code = loc_code+'-00' + if loc_code in ['DC-98','PR-98']: continue for internal_key, value in row.items(): - db.insert('census_data', seqname=False, location=loc_code, internal_key=internal_key, census_type=type, value=value) + db.insert('census_data', seqname=False, district_id=loc_code, internal_key=internal_key, census_type=type, value=value) def main(): for type in [1, 3]: @@ -61,7 +64,7 @@ if __name__ == "__main__": db = bulk_loader_db(os.environ.get('WATCHDOG_TABLE', 'watchdog_dev')) meta_cols = ['internal_key', 'census_type', 'hr_key'] db.open_table('census_meta', meta_cols, filename=tsv_file_format%'census_meta') - data_cols = ['location', 'internal_key', 'census_type', 'value'] + data_cols = ['district_id', 'internal_key', 'census_type', 'value'] db.open_table('census_data', data_cols, filename=tsv_file_format%'census_data') main() else: import/load/census.py @@ -29,14 +29,14 @@ def get_dist(zip5): if db.select('state', where='code=$zip5',vars=locals()): return {zip5:1.0} dists = db.select('zip4', - what='COUNT(plus4), district', + what='COUNT(plus4), district_id', where='zip=$zip5', - group='district', + group='district_id', vars=locals()).list() all_zip4 = sum(map(lambda d: d.count, dists)) ret = {} for d in dists: - ret[d.district] = float(d.count) / float(all_zip4) + ret[d.district_id] = float(d.count) / float(all_zip4) return ret @@ -70,12 +70,16 @@ def load_soi(): cur_data[k] = new_data[k] for d in districts.keys(): for b in districts[d]['brackets']: + if d == 'DC': d = 'DC-00' # HACK: Data has DC which should be DC-00. + # We can't use None because bracket_low is part of the primary key. if isinstance(b['bracket_low'], NoneType): b['bracket_low'] = -1 - db.insert('soi', seqname=False, location=d, **b) + db.insert('soi', seqname=False, district_id=d, **b) #pprint(districts) if __name__ == "__main__": - load_soi() + with db.transaction(): + db.delete('soi',where='1=1') + load_soi() import/load/soi.py @@ -267,6 +267,7 @@ class WYR(sql.Table): class SOI(sql.Table): + #district_id = sql.String(10, primary=True) district = sql.Reference(District, primary=True) # irs/soi bracket_low = sql.Integer(primary=True) @@ -291,8 +292,9 @@ class Census_meta(sql.Table): census_type = sql.Integer(primary=True) hr_key = sql.String(512) class Census_data(sql.Table): + #district_id = sql.String(10, primary=True) district = sql.Reference(District, primary=True) - internal_key = sql.String(10,primary=True) + internal_key = sql.String(10, primary=True) census_type = sql.Integer(primary=True) value = sql.Float() #db.query("CREATE VIEW census AS select * from census_meta NATURAL JOIN census_data") schema.py @@ -80,16 +80,16 @@ CREATE TABLE census_meta ( ); ALTER TABLE census_meta ADD primary key (internal_key, census_type); CREATE TABLE census_data ( - location varchar(10), + distric_id varchar(10), internal_key varchar(10), census_type smallint, -- 1 or 3 value numeric ); -ALTER TABLE census_data ADD primary key (location, internal_key, census_type); +ALTER TABLE census_data ADD primary key (district_id, internal_key, census_type); CREATE VIEW census AS select * from census_meta NATURAL JOIN census_data; CREATE TABLE soi ( - location varchar(10), + district_id varchar(10), -- irs/soi bracket_low int, @@ -112,7 +112,7 @@ CREATE TABLE soi ( pct_eitc numeric, pct_prepared numeric ); -alter table soi add primary key (location, bracket_low); +alter table soi add primary key (district_id, bracket_low); CREATE TABLE politician ( -- index.json schema.sql 62d17896374ade42e15e54e525c31cd4048e9c7c Jeff Thomas jeffdthomas@gmail.com http://github.com/aaronsw/watchdog/commit/d16ff0c793412ec9926896c32dc64fa2baeac713 d16ff0c793412ec9926896c32dc64fa2baeac713 2008-09-10T22:28:12-07:00 2008-09-10T14:13:11-07:00 Some fixes for merging with smartsql fdcc7beb4cf2fdbdfa03239e2c35e9b3f21ee2d0 Jeff Thomas jeffdthomas@gmail.com