Merge 1b10688 into 996614a

kingfisher_scrapy/extensions.py

@@ -3,6 +3,7 @@
 import json
 import os
 
+import sentry_sdk
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
 
@@ -162,3 +163,21 @@ def _request(self, item, spider, method, *args, name='API'):
         if not response.ok:
             spider.logger.warning(
                 'Failed to post [{}]. {} status code: {}'.format(item['url'], name, response.status_code))
+
+
+# https://stackoverflow.com/questions/25262765/handle-all-exception-in-scrapy-with-sentry
+class SentryLogging:
+    """
+    Sends exceptions and log records to Sentry. Log records with a level of ``ERROR`` or higher are captured as events.
+
+    https://docs.sentry.io/platforms/python/logging/
+    """
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        sentry_dsn = crawler.settings.get('SENTRY_DSN', None)
+        if sentry_dsn is None:
+            raise NotConfigured
+        extension = cls()
+        sentry_sdk.init(sentry_dsn)
+        return extension

kingfisher_scrapy/settings.py

@@ -70,6 +70,7 @@
 #    'scrapy.extensions.telnet.TelnetConsole': None,
 #}
 EXTENSIONS = {
+    'kingfisher_scrapy.extensions.SentryLogging': -1,
     # `KingfisherFilesStore` must run before `KingfisherProcessAPI`, because the file needs to be written before the
     # request is sent to Kingfisher Process.
     'kingfisher_scrapy.extensions.KingfisherFilesStore': 100,
@@ -82,6 +83,10 @@
    'kingfisher_scrapy.pipelines.Validate': 300,
 }
 
+
+# To send exceptions and log records to Sentry.
+SENTRY_DSN = os.getenv('SENTRY_DSN')
+
 # To send items to Kingfisher Process, see
 # https://kingfisher-collect.readthedocs.io/en/latest/kingfisher_process.html
 KINGFISHER_API_URI = os.getenv('KINGFISHER_API_URI')

requirements.in

@@ -6,4 +6,5 @@ rarfile
 requests
 Scrapy
 scrapyd-client
-ijson>=3
+ijson>=3
+sentry-sdk

requirements.txt

@@ -6,7 +6,7 @@
 #
 attrs==19.3.0             # via automat, service-identity, twisted
 automat==0.8.0            # via twisted
-certifi==2019.11.28       # via requests
+certifi==2019.11.28       # via requests, sentry-sdk
 cffi==1.13.2              # via cryptography
 chardet==3.0.4            # via requests
 constantly==15.1.0        # via twisted
@@ -31,10 +31,11 @@ rarfile==3.1
 requests==2.22.0
 scrapy==1.8.0
 scrapyd-client==1.1.0
+sentry-sdk==0.14.4
 service-identity==18.1.0  # via scrapy
 six==1.13.0               # via automat, cryptography, parsel, protego, pyhamcrest, pyopenssl, scrapy, scrapyd-client, w3lib
 twisted==20.3.0           # via scrapy
-urllib3==1.25.7           # via requests
+urllib3==1.25.7           # via requests, sentry-sdk
 w3lib==1.21.0             # via parsel, scrapy
 zope.interface==4.7.1     # via scrapy, twisted
 

requirements_dev.txt

@@ -50,6 +50,7 @@ rarfile==3.1
 requests==2.22.0
 scrapy==1.8.0
 scrapyd-client==1.1.0
+sentry-sdk==0.14.4
 service-identity==18.1.0
 six==1.13.0
 twisted==20.3.0