Merge pull request #215 from Lukas0907/next

Spider fixes
PyFeeds · May 16, 2020 · 02fc464 · 02fc464
2 parents fc23b1f + 34eaddd
commit 02fc464
Show file tree

Hide file tree

Showing 15 changed files with 73 additions and 305 deletions.
diff --git a/docs/development.rst b/docs/development.rst
@@ -111,6 +111,7 @@ Utilizing an API
 Some use a REST API which we can use to fetch the content.
 
   * :ref:`spider_falter.at`
+  * :ref:`spider_indiehackers.com`
   * :ref:`spider_kurier.at`
   * :ref:`spider_oe1.orf.at`
   * :ref:`spider_tvthek.orf.at`
@@ -120,7 +121,6 @@ Utilizing the sitemap
 ~~~~~~~~~~~~~~~~~~~~~
 Others provide a sitemap_ which we can parse:
 
-  * :ref:`spider_diepresse.com`
   * :ref:`spider_profil.at`
 
 Custom extraction
@@ -134,8 +134,6 @@ scraping from there.
   * :ref:`spider_cbird.at`
   * :ref:`spider_delinski.at`
   * :ref:`spider_flimmit.com`
-  * :ref:`spider_help.gv.at`
-  * :ref:`spider_indiehackers.com`
   * :ref:`spider_openwrt.org`
   * :ref:`spider_puls4.com`
   * :ref:`spider_python-patterns.guide`

diff --git a/docs/spiders/diepresse.com.rst b/docs/spiders/diepresse.com.rst
diff --git a/docs/spiders/help.gv.at.rst b/docs/spiders/help.gv.at.rst
diff --git a/docs/spiders/spotify.com.rst b/docs/spiders/spotify.com.rst
@@ -14,6 +14,10 @@ Add ``spotify.com`` to the list of spiders:
    spiders =
      spotify.com
 
+The market you are in (i. e. your country as an ISO 3166-1 alpha-2 country
+code) has to be specified in the config as well. For example, for Austria
+specify: ``market = AT``
+
 spotify.com supports different podcasts via the ``show`` parameter (one per
 line).
 
@@ -22,5 +26,6 @@ Example configuration:
 .. code-block:: ini
 
  [spotify.com]
+ market = AT
  shows =
      6u7pI0o0CUBQq0T1fwPgbj
diff --git a/feeds.cfg.dist b/feeds.cfg.dist
@@ -142,10 +142,6 @@ useragent = feeds (+https://github.com/pyfeeds/pyfeeds)
 #links =
 #   /katalog?sortiertnach=neueste
 
-#[diepresse.com]
-#sections =
-#   Meinung/Pizzicato
-
 #[kurier.at]
 #channels =
 #    /chronik/wien
@@ -158,6 +154,7 @@ useragent = feeds (+https://github.com/pyfeeds/pyfeeds)
 #    barbara.kaufmann
 
 #[spotify.com]
+#market = AT
 #shows =
 #    6u7pI0o0CUBQq0T1fwPgbj
 

diff --git a/feeds/spiders/__init__.py b/feeds/spiders/__init__.py
@@ -13,8 +13,8 @@ def feed_headers(self):
             link=getattr(self, "feed_link", None),
             path=getattr(self, "path", None),
             author_name=getattr(self, "author_name", None),
-            icon=getattr(self, "icon", None),
-            logo=getattr(self, "logo", None),
+            icon=getattr(self, "feed_icon", None),
+            logo=getattr(self, "feed_logo", None),
         )
 
     def start_requests(self):

diff --git a/feeds/spiders/derstandard_at.py b/feeds/spiders/derstandard_at.py
@@ -9,12 +9,12 @@
 
 class DerStandardAtSpider(FeedsSpider):
     name = "derstandard.at"
-    custom_settings = {"COOKIES_ENABLED": False}
 
+    _users = {}
     _titles = {}
 
     def start_requests(self):
-        self._ressorts = self.settings.get("FEEDS_SPIDER_DERSTANDARD_AT_RESSORTS")
+        self._ressorts = self.settings.get("FEEDS_SPIDER_DERSTANDARD_AT_RESSORTS", [])
         if self._ressorts:
             self._ressorts = set(self._ressorts.split())
         else:

diff --git a/feeds/spiders/diepresse_com.py b/feeds/spiders/diepresse_com.py
diff --git a/feeds/spiders/economist_com.py b/feeds/spiders/economist_com.py
@@ -9,7 +9,7 @@
 class EconomistComSpider(FeedsXMLFeedSpider):
     name = "economist.com"
     # Don't send too many requests to not trigger the bot detection.
-    custom_settings = {"COOKIES_ENABLED": False, "DOWNLOAD_DELAY": 5.0}
+    custom_settings = {"DOWNLOAD_DELAY": 5.0}
 
     _titles = {}
 

diff --git a/feeds/spiders/ft_com.py b/feeds/spiders/ft_com.py
@@ -7,7 +7,7 @@
 
 class FtComSpider(FeedsXMLFeedSpider):
     name = "ft.com"
-    custom_settings = {"COOKIES_ENABLED": False, "REFERER_ENABLED": False}
+    custom_settings = {"REFERER_ENABLED": False}
 
     _titles = {}
 

diff --git a/feeds/spiders/generic.py b/feeds/spiders/generic.py
@@ -73,7 +73,16 @@ def parse(self, response):
 
     def _parse_article(self, response):
         feed_entry = response.meta["feed_entry"]
+
         il = FeedEntryItemLoader(parent=response.meta["il"])
+        try:
+            response.text
+        except AttributeError:
+            # Response is not text (e.g. PDF, ...).
+            il.add_value("title", feed_entry.get("title"))
+            il.add_value("content_html", feed_entry.get("summary"))
+            return il.load_item()
+
         doc = Document(response.text, url=response.url)
         il.add_value("title", doc.short_title() or feed_entry.get("title"))
         summary = feed_entry.get("summary")
@@ -86,4 +95,5 @@ def _parse_article(self, response):
         except Unparseable:
             content = summary
         il.add_value("content_html", content)
+
         return il.load_item()
diff --git a/feeds/spiders/help_gv_at.py b/feeds/spiders/help_gv_at.py