Merge 9975b29 into 4e59a97

kingfisher_scrapy/extensions.py

@@ -3,6 +3,7 @@
 import json
 import os
 
+import sentry_sdk
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
 
@@ -157,3 +158,20 @@ def _request(self, item, spider, method, *args, name='API'):
         if not response.ok:
             spider.logger.warning(
                 'Failed to post [{}]. {} status code: {}'.format(item['url'], name, response.status_code))
+
+
+# https://stackoverflow.com/questions/25262765/handle-all-exception-in-scrapy-with-sentry
+class SentryLogging:
+    """
+    Send exceptions and errors to Sentry.
+    """
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        sentry_dsn = crawler.settings.get('SENTRY_DSN', None)
+        if sentry_dsn is None:
+            raise NotConfigured
+        extension = cls()
+        # by default only the errors are sent you sentry https://docs.sentry.io/platforms/python/logging/
+        sentry_sdk.init(sentry_dsn)
+        return extension

kingfisher_scrapy/settings.py

@@ -71,6 +71,8 @@
     # request is sent to Kingfisher Process.
     'kingfisher_scrapy.extensions.KingfisherFilesStore': 100,
     'kingfisher_scrapy.extensions.KingfisherProcessAPI': 500,
+    # before any other extension
+    'kingfisher_scrapy.extensions.SentryLogging': -1,
 }
 
 # Configure item pipelines
@@ -79,6 +81,8 @@
 #    'kingfisher_scrapy.pipelines.KingfisherScrapyPipeline': 300,
 #}
 
+# To send the logs to sentry
+SENTRY_DSN = os.getenv('SENTRY_DSN')
 # To send items to Kingfishet Process, set this to, for example, "http://kingfisher.example.com" (no trailing slash).
 KINGFISHER_API_URI = os.getenv('KINGFISHER_API_URI')
 # Set this to the same value as Kingfisher Process' `API_KEYS` setting.

requirements.in

@@ -5,4 +5,5 @@ rarfile
 requests
 Scrapy
 scrapyd-client
-ijson>=3
+ijson>=3
+sentry-sdk

requirements.txt

@@ -6,10 +6,9 @@
 #
 attrs==19.3.0             # via automat, service-identity, twisted
 automat==0.8.0            # via twisted
-certifi==2019.11.28       # via requests
+certifi==2019.11.28       # via requests, sentry-sdk
 cffi==1.13.2              # via cryptography
 chardet==3.0.4            # via requests
-click==7.1.2              # via pip-tools
 constantly==15.1.0        # via twisted
 cryptography==2.8         # via pyopenssl, scrapy, service-identity
 cssselect==1.1.0          # via parsel, scrapy
@@ -19,7 +18,6 @@ ijson==3.0.3
 incremental==17.5.0       # via twisted
 lxml==4.4.2               # via parsel, scrapy
 parsel==1.5.2             # via scrapy
-pip-tools==5.1.0
 protego==0.1.16           # via scrapy
 pyasn1-modules==0.2.7     # via service-identity
 pyasn1==0.4.8             # via pyasn1-modules, service-identity
@@ -32,13 +30,13 @@ rarfile==3.1
 requests==2.22.0
 scrapy==1.8.0
 scrapyd-client==1.1.0
+sentry-sdk==0.14.4
 service-identity==18.1.0  # via scrapy
-six==1.13.0               # via automat, cryptography, parsel, pip-tools, protego, pyhamcrest, pyopenssl, scrapy, scrapyd-client, w3lib
+six==1.13.0               # via automat, cryptography, parsel, protego, pyhamcrest, pyopenssl, scrapy, scrapyd-client, w3lib
 twisted==20.3.0           # via scrapy
-urllib3==1.25.7           # via requests
+urllib3==1.25.7           # via requests, sentry-sdk
 w3lib==1.21.0             # via parsel, scrapy
 zope.interface==4.7.1     # via scrapy, twisted
 
 # The following packages are considered to be unsafe in a requirements file:
-# pip
 # setuptools

requirements_dev.txt

@@ -9,7 +9,7 @@ automat==0.8.0
 certifi==2019.11.28
 cffi==1.13.2
 chardet==3.0.4
-click==7.1.2
+click==7.1.2              # via pip-tools
 constantly==15.1.0
 coverage==5.0.3           # via coveralls, pytest-cov
 coveralls==2.0.0
@@ -49,6 +49,7 @@ rarfile==3.1
 requests==2.22.0
 scrapy==1.8.0
 scrapyd-client==1.1.0
+sentry-sdk==0.14.4
 service-identity==18.1.0
 six==1.13.0
 twisted==20.3.0

tests/test_extensions.py

@@ -6,7 +6,7 @@
 import pytest
 from scrapy.exceptions import NotConfigured
 
-from kingfisher_scrapy.extensions import KingfisherProcessAPI, KingfisherFilesStore
+from kingfisher_scrapy.extensions import KingfisherFilesStore, KingfisherProcessAPI
 from kingfisher_scrapy.items import FileError
 from tests import spider_with_crawler