Merge pull request #62 from pingihu/master

Add timeouts for fetching and wrangling posts

baleen/config.py

@@ -63,6 +63,7 @@ class BaleenConfiguration(confire.Configuration):
     logfile    = 'baleen.log'                    # Location to write log
     loglevel   = 'DEBUG'                         # Log messages to record
     fetch_html = True                            # Actually fetch HTML link
+    timeout    = 180                             # Timeout for fetching posts/feeds
 
 ## Load settings immediately for import
 settings = BaleenConfiguration.load()

baleen/exceptions.py

@@ -64,3 +64,10 @@ class ExportError(BaleenError):
     Something went wrong with the export of the corpus
     """
     pass
+
+
+class TimeoutError(Exception):
+    """
+    An operation timed out
+    """
+    pass

baleen/feed.py

@@ -19,11 +19,12 @@
 
 import feedparser
 
+from baleen.config import settings
 from baleen.models import Feed
 from baleen.utils.timez import localnow
 from baleen.exceptions import FeedTypeError
 from baleen.exceptions import SynchronizationError
-from baleen.utils.decorators import memoized, reraise
+from baleen.utils.decorators import memoized, reraise, timeout
 
 
 ##########################################################################
@@ -103,6 +104,7 @@ def url(self):
             self.MODEL: lambda: self.feed.link,
         }[self.type]()
 
+    @timeout(settings.timeout)
     def parse(self):
         """
         Wraps the feedparser.parse function such that if the feed is an model,

baleen/utils/decorators.py

@@ -17,9 +17,10 @@
 ## Imports
 ##########################################################################
 
+import signal
 from functools import wraps
 from baleen.utils.timez import Timer
-from baleen.exceptions import BaleenError
+from baleen.exceptions import BaleenError, TimeoutError
 
 ##########################################################################
 ## Memoization
@@ -65,6 +66,30 @@ def timer_wrapper(*args, **kwargs):
     return timer_wrapper
 
 
+def timeout(seconds):
+    """
+    Raises a TimeoutError if a function does not terminate within
+    specified seconds.
+    """
+    def _timeout_error(signal, frame):
+        raise TimeoutError("Operation did not finish within \
+        {} seconds".format(seconds))
+
+    def timeout_decorator(func):
+
+        @wraps(func)
+        def timeout_wrapper(*args, **kwargs):
+            signal.signal(signal.SIGALRM, _timeout_error)
+            signal.alarm(seconds)
+            try:
+                return func(*args, **kwargs)
+            finally:
+                signal.alarm(0)
+
+        return timeout_wrapper
+
+    return timeout_decorator
+
 ##########################################################################
 ## Exception Handling
 ##########################################################################

baleen/wrangle.py

@@ -26,6 +26,7 @@
 from copy import deepcopy
 from dateutil import parser as dtparser
 
+from baleen.config import settings
 from baleen.models import Post
 from baleen.utils.decorators import reraise
 from baleen.exceptions import WranglingError, FetchError
@@ -163,7 +164,7 @@ def fetch(self, save=True):
         if not self.is_wrangled():
             raise ValueError("Entry not yet wrangled, cannot fetch.")
 
-        response = requests.get(self.post.url)
+        response = requests.get(self.post.url, timeout=settings.timeout)
         response.raise_for_status()
 
         if response.text: