Permalink
Browse files

More error handling in HTTP requests

  • Loading branch information...
1 parent 9827f6b commit 73c4575954a631b5741beb7ef6d009e89272ff5d @brutasse brutasse committed Mar 18, 2013
Showing with 33 additions and 4 deletions.
  1. +12 −3 feedhq/feeds/models.py
  2. +21 −1 tests/test_feeds.py
View
@@ -21,6 +21,7 @@
from django.utils.translation import ugettext_lazy as _
from django_push.subscriber.signals import updated
+from httplib import IncompleteRead
from lxml.etree import ParserError
from requests.packages.urllib3.exceptions import LocationParseError
@@ -162,14 +163,18 @@ def update_feed(self, url, use_etags=True):
try:
response = requests.get(url, headers=headers,
timeout=obj.request_timeout)
- except (requests.RequestException, socket.timeout) as e:
+ except (requests.RequestException, socket.timeout, socket.error,
+ IncompleteRead) as e:
logger.debug("Error fetching %s, %s" % (obj.url, str(e)))
if obj.backoff_factor == obj.MAX_BACKOFF - 1:
logger.debug(
- "%s reached max backoff period (timeout)" % obj.url
+ "%s reached max backoff period (%s)" % (obj.url, str(e))
)
obj.backoff()
- obj.error = obj.TIMEOUT
+ if isinstance(e, IncompleteRead):
+ obj.error = obj.CONNECTION_ERROR
+ else:
+ obj.error = obj.TIMEOUT
if save:
obj.save(update_fields=['backoff_factor', 'error'])
return
@@ -280,6 +285,7 @@ class UniqueFeed(models.Model):
GONE = 'gone'
TIMEOUT = 'timeout'
PARSE_ERROR = 'parseerror'
+ CONNECTION_ERROR = 'connerror'
HTTP_400 = '400'
HTTP_401 = '401'
HTTP_403 = '403'
@@ -291,6 +297,7 @@ class UniqueFeed(models.Model):
(GONE, 'Feed gone (410)'),
(TIMEOUT, 'Feed timed out'),
(PARSE_ERROR, 'Location parse error'),
+ (CONNECTION_ERROR, 'Connection error'),
(HTTP_400, 'HTTP 400'),
(HTTP_401, 'HTTP 401'),
(HTTP_403, 'HTTP 403'),
@@ -598,6 +605,8 @@ def update_favicon(self, link, force_update=False):
page = requests.get(link, headers=ua, timeout=10).content
except requests.RequestException:
return favicon
+ except LocationParseError:
+ return favicon
if not page:
return favicon
View
@@ -3,6 +3,7 @@
import json
import os
+from httplib import IncompleteRead
from io import StringIO, BytesIO
from django_push.subscriber.signals import updated
@@ -88,7 +89,21 @@ def test_parse_error(self, get):
UniqueFeed.objects.update_feed(f.url)
f = UniqueFeed.objects.get()
self.assertTrue(f.muted)
- self.assertEqual(f.error, 'parseerror')
+ self.assertEqual(f.error, f.PARSE_ERROR)
+
+ @patch("requests.get")
+ def test_incomplete_read(self, get):
+ get.side_effect = IncompleteRead("0 bytes read")
+ u = User.objects.create_user('foobar', 'test@example.com', 'pass')
+ c = Category.objects.create(name='foo', slug='foo', user=u)
+ f = Feed.objects.create(
+ url='http://www.rubycocoa.com/syndicate/rss/feed.xml',
+ category=c,
+ )
+ UniqueFeed.objects.update_feed(f.url)
+ f = UniqueFeed.objects.get()
+ self.assertFalse(f.muted)
+ self.assertEqual(f.error, f.CONNECTION_ERROR)
class TestFeeds(TestCase):
@@ -918,3 +933,8 @@ class Response:
headers = {}
get.return_value = Response()
Favicon.objects.update_favicon('http://example.com')
+
+ @patch("requests.get")
+ def test_favicon_parse_error(self, get):
+ get.side_effect = LocationParseError("Failed to parse url")
+ Favicon.objects.update_favicon('http://example.com')

0 comments on commit 73c4575

Please sign in to comment.