Skip to content

Commit

Permalink
Revert "MA: Make vote scrape smarter about continuing despite the occ…
Browse files Browse the repository at this point in the history
…assional 404 or other error."

This reverts commit 8c7b069.

This commit put the scraper into an unending loop attempting to download vote pdfs that didn't exist.
  • Loading branch information
twneale committed Jul 29, 2014
1 parent 8b90bb6 commit db8fbc1
Showing 1 changed file with 4 additions and 12 deletions.
16 changes: 4 additions & 12 deletions openstates/ma/votes.py
Expand Up @@ -48,7 +48,6 @@ class MiscellaneousVote(Exception):
'''

def scrape(self, chamber, session):
self._404_count = 0
self.filenames = []
if chamber == 'upper':
self.scrape_senate(session)
Expand Down Expand Up @@ -82,17 +81,10 @@ def scrape_vote(self, session, rollcall_number):

try:
vote_file, resp = self.urlretrieve(url)
except scrapelib.HTTPError as exc:
if exc.response.status_code is 404:
if 10 < self._404_count:
# We'll hit a number of 404s at the end of the votes.
self.warning('Stopping; encountered a 404 at %s' % url)
raise self.EndOfHouseVotes()
else:
self._404_count += 1
else:
self.logger.exception(exc)
continue
except scrapelib.HTTPError:
# We'll hit a 404 at the end of the votes.
self.warning('Stopping; encountered a 404 at %s' % url)
raise self.EndOfHouseVotes

text = convert_pdf(vote_file, type='text')
text = text.decode('utf8')
Expand Down

0 comments on commit db8fbc1

Please sign in to comment.