Merge pull request #192 from Lukas0907/next

Fixes for site changes

feeds/loaders.py

@@ -22,9 +22,7 @@
 
 logger = logging.getLogger(__name__)
 
-_lxml_cleaner = Cleaner(
-    scripts=True, javascript=True, comments=True, style=True, inline_style=True
-)
+_lxml_cleaner = Cleaner(style=True)
 
 # List of so-called empty elements in HTML.
 # Source: https://developer.mozilla.org/en-US/docs/Glossary/Empty_element

feeds/spiders/arstechnica_com.py

@@ -1,3 +1,5 @@
+import re
+
 import scrapy
 
 from feeds.loaders import FeedEntryItemLoader
@@ -53,10 +55,33 @@ def parse_node(self, response, node):
             meta={"il": il, "path": response.meta["path"], "first_page": True},
         )
 
+    @staticmethod
+    def _div_to_img(elem):
+        elem.tag = "img"
+        url = re.search(r"url\('([^']+)'\)", elem.attrib["style"]).group(1)
+        elem.attrib["src"] = url
+        elem.attrib["style"] = None
+        return elem
+
     def _parse_article(self, response):
-        remove_elems = [".caption-credit", ".gallery-image-credit"]
+        remove_elems = [
+            ".caption-credit",
+            ".gallery-image-credit",
+            "#social-left",
+            "ul.toc",
+            "h3:contains('Table of Contents')",
+            "br",
+            ".sidebar:contains('Further Reading')",
+            ".credit",
+        ]
+        change_tags = {".sidebar": "blockquote", "aside": "blockquote"}
+        replace_elems = {"div.image": self._div_to_img}
         il = FeedEntryItemLoader(
-            response=response, parent=response.meta["il"], remove_elems=remove_elems
+            response=response,
+            parent=response.meta["il"],
+            remove_elems=remove_elems,
+            replace_elems=replace_elems,
+            change_tags=change_tags,
         )
         if response.meta.get("first_page", False):
             il.add_value("link", response.url)

feeds/spiders/tvthek_orf_at.py

@@ -4,6 +4,7 @@
 from dateutil.tz import gettz
 from scrapy import Request
 
+from feeds.exceptions import DropResponse
 from feeds.loaders import FeedEntryItemLoader
 from feeds.spiders import FeedsSpider
 
@@ -85,7 +86,14 @@ def _parse_episode(self, response):
             )
             il.add_value("enclosure", {"iri": video["src"], "type": "video/mp4"})
         except StopIteration:
-            self.logger.error("Could not extract video for '{}'!".format(item["title"]))
+            self.logger.warning(
+                "Could not extract video for '{}'!".format(item["title"])
+            )
+            raise DropResponse(
+                "Skipping {} because not downloadable yet".format(response.url),
+                transient=True
+            )
+
         subtitle = item["_embedded"].get("subtitle")
         if subtitle:
             subtitle = subtitle["_embedded"]["srt_file"]["public_urls"]["reference"]

feeds/spiders/ubup_com.py

@@ -45,6 +45,11 @@ def parse(self, response):
             image_url = item.css(".item-image::attr(data-bg)").re_first(
                 r"url\(([^)]+)\)"
             )
+            # Fix broken images.
+            if image_url.startswith("https://markenankauf.momox.de/pics/https://"):
+                image_url = image_url.replace(
+                    "https://markenankauf.momox.de/pics/https://", "https://"
+                )
             il.add_value("content_html", '<img src="{}">'.format(image_url))
             il.add_css("content_html", ".item-des-container")
             il.add_value("path", response.meta["path"])