Skip to content
Permalink
Browse files

Catch exception when getting page

  • Loading branch information...
katkad committed Oct 17, 2016
1 parent 4367c06 commit 364b0415c613538a04abe9ec9520c66400ee3d17
Showing with 8 additions and 1 deletion.
  1. +8 −1 scraper.py
@@ -108,7 +108,14 @@ def get_content(self, path):
'''
url = urlparse.urljoin(self.DOMAIN, path)
logging.info('Requesting content from url: "{}"'.format(url))
response = requests.get(url)

response = ''
try:
response = requests.get(url)
except Exception as e:
logging.error('Could not load page "{}" (ERROR: {})'.format(url, format(e)))
time.sleep(5)
return None

if response.status_code not in self.HTTP_OK_CODES:
logging.error('Could not load page "{}" (CODE: {})'.format(url, response.status_code))

0 comments on commit 364b041

Please sign in to comment.
You can’t perform that action at this time.