Skip to content

Commit

Permalink
add make .crawlwyr
Browse files Browse the repository at this point in the history
  • Loading branch information
Aaron Swartz committed Feb 14, 2012
1 parent e0ad9b0 commit 2e56abc
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 50 deletions.
4 changes: 4 additions & 0 deletions import/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,10 @@ $(LOAD)/politicians/index.json: load/manual/politicians.json
$(PYTHON) load/votesmart.py
touch $@

.crawlwyr: crawl/wyr.py
$(PYTHON) crawl/wyr.py
touch $@

.wyr: .schema load/wyr.py $(CRAWL)/votesmart/wyr.json
$(PYTHON) load/wyr.py
touch $@
Expand Down
21 changes: 16 additions & 5 deletions import/crawl/wyr.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import sys
import simplejson as json
try:
import json
except ImportError:
import simplejson as json
from ClientForm import ParseResponse, ControlNotFoundError, AmbiguityError
from settings import db
from BeautifulSoup import BeautifulSoup
Expand Down Expand Up @@ -83,11 +86,14 @@ def getzip(dist):

def has_wyr(pol):
dist = pol2dist(pol)
return has_wyr_dist(dist)

def has_wyr_dist(dist):
if len(dist) == 2: return False # senators don't have forms in WYR system

try:
response = urlopen(WYR_URL)
form = ParseResponse(response, backwards_compat=False)[1] #first form is of search
form = ParseResponse(response, backwards_compat=False)[2] #1st form is of search, 2nd is findrep
except:
return False

Expand All @@ -100,6 +106,7 @@ def has_wyr(pol):
form['zip'] = zip5
form['zip4'] = zip4
request = form.click()
return request.get_full_url()
try:
response = urlopen(request.get_full_url(), request.get_data())
forms = ParseResponse(response, backwards_compat=False)
Expand Down Expand Up @@ -224,7 +231,7 @@ def get_votesmart_contacts(pols):
if not _url and addr['webAddressType'] == 'Webmail':
url = addr['webAddress']
_url, contacttype = getformtype(url)
if contacttype and contacttype != 'wyr':
captcha = (contacttype == 'ima') and has_captcha(url)
d[r.id] = dict(contact=_url, contacttype=contacttype, captcha=captcha)
Expand Down Expand Up @@ -285,6 +292,10 @@ def main(fname='../data/crawl/votesmart/wyr.json'):

f = file(fname, 'w')
json.dump(d, f, indent=2, sort_keys=True)


def main2():
for dist in dist_zip_dict:
print dist, has_wyr_dist(dist)

if __name__ == '__main__':
main()
main2()
86 changes: 41 additions & 45 deletions templates/base.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<!-- Google Website Optimizer Control Script -->
<script>
function utmx_section(){}function utmx(){}
(function(){var k='1615833661',d=document,l=d.location,c=d.cookie;function f(n){
(function(){var k='3839909263',d=document,l=d.location,c=d.cookie;function f(n){
if(c){var i=c.indexOf(n+'=');if(i>-1){var j=c.indexOf(';',i);return c.substring(i+n.
length+1,j<0?c.length:j)}}}var x=f('__utmx'),xx=f('__utmxx'),h=l.hash;
d.write('<sc'+'ript src="'+
Expand All @@ -13,10 +14,10 @@
+new Date().valueOf()+(h?'&utmxhash='+escape(h.substr(1)):'')+
'" type="text/javascript" charset="utf-8"></sc'+'ript>')})();
</script>
<!-- End of Google Website Optimizer Control Script -->
$if page.get('front'):
<title>watchdog.net: $:page.title.strip()</title>
<meta name="verify-v1" content="V7KJtoWCZfItupnGhaoUY6gwX1aVkrFszhCOoM0qgbg=" />
<meta name="verify-v1" content="H+m5dkTl7hRG+Yp3hrB/2XuoZO4O1TAaq7IxTwRzjpk=" />
<meta name="google-site-verification" content="A_o8iekZ1hC0lpGVuhUl1V18YAk4XjA69V28vSZvMH4" />
$else:
<title>$:page.title.strip() (watchdog.net)</title>

Expand Down Expand Up @@ -139,6 +140,18 @@
//-->
</script>
</div>
<script type="text/javascript"><!--
google_ad_client = "pub-4591221763003403";
/* 728x90, top */
google_ad_slot = "3411769551";
google_ad_width = 728;
google_ad_height = 90;

if (document.wd_first_time) {
document.write('<scr' + 'ipt type="text/javascript" src="http://pagead2.googlesyndication.com/pagead/show_ads.js"></scr' + 'ipt>');
}
//-->
</script>

$:page.get('pretext')

Expand Down Expand Up @@ -195,73 +208,55 @@
$if not page.get('front'):-leaf\
">
<div id="left_footer">
<form action="http://salsa.wiredforchange.com/dia/processEditValues.jsp" method="post" >
<form method="POST" action="http://act.demandprogress.org/act/" accept-charset="utf-8">
<input type="hidden" name="utf8" value="&#x2714;">
<input type="hidden" name="redirect" value="http://watchdog.net/thanks?url=$urlquote(getfullpath())" />
<input type="hidden" name="table" value="supporter"/>
<input type="hidden" name="tag" value="wd_bottom1" />
<input type="hidden" name="Receive_Email" value="3" />
<input type="hidden" name="organization_KEY" value="5649" />
<input type="hidden" name="link" value="custom" />
<input type="hidden" name="linkKey" value="0" />
<input type="hidden" name="link" value="groups" />
<input type="hidden" name="linkKey" value="40117" />
<input type="hidden" name="link" value="groups" />
<input type="hidden" name="linkKey" value="39103" />
<input type="hidden" name="link" value="groups" />
<input type="hidden" name="linkKey" value="38203" />
<input type="hidden" name="lists" value="1" />
<input type="hidden" name="page" value="watchdog" />
<input type="hidden" name="source" value="bottom" />
<input type="hidden" name="action_path" value="$getfullpath()" />
<strong>Help make a difference:</strong>
Sign up for our occasional email list:<br />
<input type="text" name="First_Name" value="first name"
<input type="text" name="first_name" value="first name"
onclick="clickclear(this, 'first name')"
onblur="clickrecall(this, 'first name')"/>
<input type="text" name="Last_Name" value="last name"
<input type="text" name="last_name" value="last name"
onclick="clickclear(this, 'last name')"
onblur="clickrecall(this, 'last name')" /><br />
<input type="text" name="Email" value="email"
<input type="text" name="email" value="email"
onclick="clickclear(this, 'email')"
onblur="clickrecall(this, 'email')" />
<input type="text" name="Zip" value="zip code"
<input type="text" name="zip" value="zip code"
onclick="clickclear(this, 'zip code')"
onblur="clickrecall(this,'zip code')" />
<button type="submit">Go</button>
</form>
</div>
<div id="dialog">
<form name="dialogform" action="http://salsa.wiredforchange.com/dia/processEditValues.jsp" method="post" onsubmit="return validate(this);">
<form method="POST" action="http://act.demandprogress.org/act/" accept-charset="utf-8">
<input type="hidden" name="utf8" value="&#x2714;">
<input type="hidden" name="redirect" value="http://watchdog.net/thanks?url=$urlquote(getfullpath())" />
<input type="hidden" name="table" value="supporter"/>
<input type="hidden" name="tag" value="wd_dialog1" />
<input type="hidden" name="Receive_Email" value="3" />
<input type="hidden" name="organization_KEY" value="5649" />
<input type="hidden" name="link" value="custom" />
<input type="hidden" name="linkKey" value="0" />
<input type="hidden" name="link" value="groups" />
<input type="hidden" name="linkKey" value="40117" />
<input type="hidden" name="link" value="groups" />
<input type="hidden" name="linkKey" value="39103" />
<input type="hidden" name="link" value="groups" />
<input type="hidden" name="linkKey" value="38203" />
<input type="hidden" name="lists" value="1" />
<input type="hidden" name="page" value="watchdog" />
<input type="hidden" name="source" value="dialog" />
<input type="hidden" name="action_path" value="$getfullpath()" />
<p>
<script>utmx_section("Headline")</script>
<script>utmx_section("Intro")</script>
<strong>Help make a difference:</strong>
</noscript>
<script>utmx_section("Body")</script>
Sign up for our occasional email list:
</noscript></p>
<p id="dialog_error" style="color: red"></p>
<p>
<script>utmx_section("Name ask")</script>
<input type="text" name="First_Name" value="first name"
<input type="text" name="first_name" value="first name"
onclick="clickclear(this, 'first name')"
onblur="clickrecall(this, 'first name')"/>
<input type="text" name="Last_Name" value="last name"
<input type="text" name="last_name" value="last name"
onclick="clickclear(this, 'last name')"
onblur="clickrecall(this, 'last name')" /><br />
</noscript>
<input type="text" name="Email" value="email"
<input type="text" name="email" value="email"
onclick="clickclear(this, 'email')"
onblur="clickrecall(this, 'email')" />
<input type="text" name="Zip" value="zip code"
<input type="text" name="zip" value="zip code"
onclick="clickclear(this, 'zip code')"
onblur="clickrecall(this,'zip code')" />
<button type="submit">Go</button>
Expand All @@ -282,13 +277,14 @@
var pageTracker = _gat._getTracker("UA-7198052-1");
pageTracker._trackPageview();
} catch(err) {}</script>
<!-- Google Website Optimizer Tracking Script -->
<script type="text/javascript">
if(typeof(_gat)!='object')document.write('<sc'+'ript src="http'+
(document.location.protocol=='https:'?'s://ssl':'://www')+
'.google-analytics.com/ga.js"></sc'+'ript>')</script>
<script type="text/javascript">
try {
var pageTracker=_gat._getTracker("UA-7198052-2");
pageTracker._trackPageview("/1615833661/test");
var gwoTracker=_gat._getTracker("UA-7198052-2");
gwoTracker._trackPageview("/3839909263/test");
}catch(err){}</script>
</html>
<!-- End of Google Website Optimizer Tracking Script -->

0 comments on commit 2e56abc

Please sign in to comment.