Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Branch: master
Fetching contributors…

Cannot retrieve contributors at this time

80 lines (58 sloc) 2.413 kB
"""Not currently used in this app, but scrapes the schema of tables from FEC's web site and generates django models."""
import re
import urllib2
from BeautifulSoup import BeautifulSoup
re_char = re.compile('VARCHAR2\s*\((\d+)(?: Byte)?\)')
def getdatatype(fec):
if re_char.match(fec):
return 'Char', re_char.match(fec).groups()[0]
if re.match('NUMBER\s*\(\d+\)',fec):
return 'Integer',None
if re.match('NUMBER\s*\(\d+,\d\)',fec):
return 'Float',None
if re.match('DATE',fec):
return 'Date',None
return 'unknown',None
def unsoupify(cell):
if not cell.contents or not cell.contents[0]:
return ''
return unicode(cell.contents[0]).strip().replace('"','')
tables = {
'cm': 'CommitteeMaster',
'cn': 'CandidateMaster',
'ccl': 'CandCmteLinkage',
'indiv': 'ContributionsbyIndividuals',
'pas': 'ContributionstoCandidates',
'oth': 'CommitteetoCommittee',
'weball': 'WEBALL',
'weballk': 'WEBK',
}
def writemodels():
fout = open('models.py','w')
fout.write("from django.db import models\n\n\n")
for key in tables.keys():
html = urllib2.urlopen('http://www.fec.gov/finance/disclosure/metadata/DataDictionary%s.shtml' % tables[key]).read()
soup = BeautifulSoup(html).table.findAll('tr')[1:]
fout.write("""
class %s(models.Model):
class Meta:
verbose_name="%s"
cycle = models.IntegerField()
""" % (key, tables[key]))
for row in soup:
cells = row.findAll('td')
(name,longname,start,isnull,datatype,descrip) = [unsoupify(x) for x in cells]
(djangotype,length) = getdatatype(datatype)
if djangotype=='Char':
fout.write("""
%s = models.CharField(max_length=%s, blank=True, verbose_name="%s", help_text = "%s")""" % (name,length,longname,descrip))
elif djangotype=='Date':
fout.write("""
%s = models.DateField(blank=True, verbose_name="%s", help_text = "%s")""" % (name,longname,descrip))
elif djangotype=='Integer':
fout.write("""
%s = models.IntegerField(verbose_name="%s", help_text = "%s")""" % (name,longname,descrip))
elif djangotype=='Float':
fout.write("""
%s = models.FloatField(verbose_name="%s", help_text = "%s")""" % (name,longname,descrip))
writemodels()
Jump to Line
Something went wrong with that request. Please try again.