templates/exempt_org.html @@ -30,6 +30,7 @@ def string(s): def state(s): s = string(s) + if s == '.': return None assert s == s.upper() assert s.isalpha() return s @@ -89,7 +90,7 @@ FIELD_TYP = 2 ## The functions you might want to call -def parse_line(linedef, line): +def parse_line(linedef, line, debug=False): out = storage() n = 0 for (k, l, t) in linedef: @@ -98,8 +99,9 @@ def parse_line(linedef, line): if k is None: t(line[n:n+l]) else: + if debug: print k, repr(line[n:n+l]), out[k] = t(line[n:n+l]) - #print k, repr(line[n:n+l]) + if debug: print repr(out[k]) if l > 0: n += l return out @@ -112,7 +114,7 @@ def get_len(filedef): else: return sum(line[FIELD_LEN] for line in filedef) -def parse_file(filedef, fh, f_whichdef=None): +def parse_file(filedef, fh, f_whichdef=None, debug=False): linelen = get_len(filedef) if isinstance(filedef, dict): if not f_whichdef: f_whichdef = lambda x: x[0] @@ -120,4 +122,4 @@ def parse_file(filedef, fh, f_whichdef=None): f_whichdef = lambda x: slice(None, None) for line in iter(lambda: fh.read(linelen), ''): if line.replace('\x00', '').strip(): - yield parse_line(filedef[f_whichdef(line)], line) + yield parse_line(filedef[f_whichdef(line)], line, debug=debug) import/parse/fixed_width.py @@ -1,17 +1,18 @@ +import glob, itertools from fixed_width import integer, string, date, filler, parse_file, enum, state, digits def integer2(s): return integer(s[-1] + s[:-1]) def_eo = [ - ('_type', 0, lambda s: 'Exempt Organization'), +# ('_type', 0, lambda s: 'Exempt Organization'), ('ein', 9-0, digits), ('primary_name', 79-9, string), ('careof_name', 114-79, string), ('street', 149-114, string), ('city', 171-149, string), ('state', 173-171, state), - ('zip', 183-173, digits), - ('group_exemption_num', 187-183, integer), + ('zip', 183-173, string), + ('group_exemption_num', 187-183, string), ('subsection_code', 189-187, string), ('affiliation', 1, enum), ('classification_code', 194-190, string), @@ -34,11 +35,16 @@ def_eo = [ ('ntee_code', 282-278, string), ('sort_name', 318-282, string), (None, 2, filler('\r\n')) - ] +def parse(): + return itertools.chain(*[parse_file(def_eo, file(fn)) for fn in glob.glob('../data/crawl/irs/eo/*.LST')]) + if __name__ == "__main__": - import glob - import tools - for fn in glob.glob('../data/crawl/irs/eo/*.LST'): - tools.export(parse_file(def_eo, file(fn))) + import sys + if 'load' in sys.argv: + from settings import db + db.multiple_insert("exempt_org", parse(), seqname=False) + else: + import tools + tools.export(parse()) import/parse/irs_eo.py @@ -619,6 +619,36 @@ class Past_Elections(sql.Table): pct_votes_received = sql.Percentage() expenditure = sql.Dollars() +class Exempt_Org(sql.Table): + ein = sql.Integer(primary=True) + primary_name = sql.String() + careof_name = sql.String() + street = sql.String() + city = sql.String() + state = sql.String(2) + zip = sql.String() + group_exemption_num = sql.String() + subsection_code = sql.String() + affiliation = sql.String() + classification_code = sql.String() + ruling_date = sql.String() + deductibility_code = sql.String() + foundation_code = sql.String() + activity_code = sql.String() + organization_code = sql.String() + exempt_org_status_code = sql.String() + advance_ruling_expiration = sql.String() + tax_period = sql.String() + asset_code = sql.String() + income_code = sql.String() + filing_requirement_code = sql.String() + accounting_period = sql.String() + asset_amt = sql.BigInteger() + income_amt = sql.BigInteger() + form_990_revenue_amt = sql.BigInteger() + ntee_code = sql.String() + sort_name = sql.String() + def init(): db.query("CREATE VIEW census AS select * from census_meta NATURAL JOIN census_data") db.query("CREATE INDEX contribution_recipient_id_idx ON contribution (recipient_id)") schema.py @@ -313,3 +313,4 @@ h2.repred { color: #EE4400; margin-bottom: 6px; } td.rule {border-top: 1px solid #ccc; border-bottom: 1px solid #ccc; } td.rule-below { border-bottom: 1px solid #ccc; } +th, td { vertical-align: top; } static/css/style.css @@ -6,6 +6,18 @@ import web from config import secret_key from settings import db +def urlify(s): + """ + >>> urlify("What the !@#$%^ is going on here!?") + 'what-the--is-going-on-here' + """ + s = s.lower() + out = [] + for k in s: + if k == " ": out.append('-') + elif k.isalpha() or k.isdigit(): out.append(k) + return ''.join(out) + def encrypt(msg, key=None): return hmac.new(key or secret_key, msg).hexdigest() utils/helpers.py @@ -59,7 +59,7 @@ def make_siteindex(urls): for item in x: sitemap_lines.append('<url><loc>http://watchdog.net%s</loc></url>' % item) sitemap_lines.append('</urlset>') - write("sitemaps/sitemap_%04d.xml.gz" % i, '\n'.join(sitemap_lines))) + write("sitemaps/sitemap_%04d.xml.gz" % i, '\n'.join(sitemap_lines)) names = ["%04d" % j for j in range(i)] timestamp = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z' utils/sitemap.py @@ -240,6 +240,7 @@ class Float(Column): class Serial(Integer): sql_type = 'serial' class Int2(Integer): sql_type = 'int2' +class BigInteger(Integer): sql_type = 'bigint' class Date(Column): sql_type = 'date' class Year(Integer): pass vendor/smartersql.py @@ -3,7 +3,7 @@ import re, sys, urllib import web -from utils import zip2rep, simplegraphs, apipublish, users, writerep, se, wyrapp, api +from utils import zip2rep, simplegraphs, apipublish, users, writerep, se, wyrapp, api, helpers import blog import petition import settings @@ -49,6 +49,7 @@ urls = ( r'/lob/o/?(.*?)', 'lob_org', r'/lob/pa/?(.*?)', 'lob_pac', r'/lob/pe/?(.*?)', 'lob_person', + r'/ein/(\d+)(/.*)?', 'ein', r'/writerep', wyrapp.app, r'/api', api.app, r'/about(/?)', 'about', @@ -671,6 +672,19 @@ class lob_person: if not p: raise web.notfound() return render.lob_person(p, limit) +class ein: + def index(self): + pass #@@ + + def GET(self, ein, slug=None): + try: + p = schema.Exempt_Org.select(where='ein=$ein', vars=locals())[0] + except IndexError: + raise web.notfound() + if slug != '/' + helpers.urlify(p.primary_name): + raise web.redirect('/ein/%s/%s' % (ein, helpers.urlify(p.primary_name))) + return render.exempt_org(p) + class politician_introduced: def index(self): #/p/(.*?)/introduced @@ -931,7 +945,7 @@ if production_mode: if __name__ == "__main__": import sys - if sys.argv[1] == 'cache': + if len(sys.argv) > 1 and sys.argv[1] == 'cache': cache_occupation(sys.argv[2]) else: app.run() webapp.py 8b378cf82015dc26fe3ea00cbac8acea581802e0 Aaron Swartz me@aaronsw.com http://github.com/aaronsw/watchdog/commit/71e2b95973b8b0f742f2744b724044815c51a6a6 71e2b95973b8b0f742f2744b724044815c51a6a6 2009-08-13T12:37:19-07:00 2009-08-13T12:37:19-07:00 add support for exempt organizations (tx Noufal Ibrahim) 1c0609cd2d36680e3c173efd2097b9f8b51d222d Aaron Swartz me@aaronsw.com