Update The Guardian and the Observer

recipes/guardian.recipe

@@ -10,8 +10,6 @@ from calibre import random_user_agent
 from calibre.web.feeds.news import BasicNewsRecipe
 from datetime import date
 
-from lxml import etree
-
 
 def classes(classes):
     q = frozenset(classes.split(' '))
@@ -50,17 +48,21 @@ class Guardian(BasicNewsRecipe):
         dict(attrs={'class': lambda x: x and 'submeta' in x.split()}),
         dict(attrs={'data-component': ['share', 'social']}),
         dict(attrs={'data-link-name': 'block share'}),
+        dict(attrs={'data-print-layout': 'hide'}),
+        dict(id=['dfp-ad--survey', 'sub-nav-root', 'the-caption', 'bannerandheader']),
+        {'for': 'the-checkbox'},
+        dict(href='#maincontent'),
+        dict(role='navigation'),
         dict(attrs={'class': lambda x: x and 'inline-expand-image' in x}),
         dict(name='a', attrs={'aria-label': lambda x: x and 'Share On' in x}),
         dict(name='a', attrs={'class': lambda x: x and 'social__action js-social__action--top' in x}),
         dict(name='div', attrs={'id': 'share-count-root'}),
         dict(attrs={'class': lambda x: x and 'modern-visible' in x.split()}),
         classes('badge-slot reveal-caption__checkbox mobile-only element-rich-link'),
-        dict(name=['link', 'meta', 'style', 'svg', 'input', 'source']),
+        dict(name=['link', 'meta', 'style', 'svg', 'input', 'source', 'noscript', 'button']),
     ]
     remove_tags_after = [
-        dict(attrs={'class': lambda x: x and 'content__article-body' in x.split()}),
-        dict(attrs={'class': lambda x: x and 'article-body-commercial-selector' in x.split()}),
+        classes('content__article-body js-bottom-marker article-body-commercial-selector'),
     ]
 
     extra_css = """
@@ -117,55 +119,3 @@ class Guardian(BasicNewsRecipe):
         if date.today().weekday() in (5, 6):
             feeds += self.parse_section('https://www.theguardian.com/theguardian/weekend', 'Weekend - ')
         return feeds
-
-    def preprocess_html(self, soup):
-        old_body = soup.find('body')
-        if old_body is not None:
-            main_column = soup.find(**classes('content__main-column'))
-            if main_column is None:
-                for section in soup.findAll('section'):
-                    if section.find('h1') is not None:
-                        main_column = section
-                        break
-            if main_column is not None:
-                body = soup.new_tag('body')
-                body.append(main_column)
-                old_body.replaceWith(body)
-
-        for img in soup.findAll('img', srcset=True):
-            img['src'] = img['srcset'].partition(' ')[0]
-            img['srcset'] = ''
-        return soup
-
-    def preprocess_raw_html(self, raw, url):
-        try:
-            root = self.index_to_soup(raw, as_tree=True)
-            # find out Share on facebook elements
-            share_platforms = root.xpath('//div//div//div/ul/li/a[starts-with(@aria-label,"Share on")]')
-            if share_platforms and len(share_platforms) > 0:
-                parent_div = share_platforms[0].getparent().getparent().getparent()
-                ul_elem = share_platforms[0].getparent().getparent()
-                # remove ul element
-                parent_div.remove(ul_elem)
-
-            share_counts = root.xpath('//div//div//div[contains(@id,"-count-root")]')
-            if share_counts and len(share_counts) > 0:
-                share_parent_div = share_counts[0].getparent().getparent()
-                # remove share count element
-                share_parent_div.remove(share_counts[0].getparent())
-
-            pub_date_div = root.xpath('//div//div//div//input[@id="dateToggle"]')
-            if pub_date_div:
-                input_elem = pub_date_div[0]
-                date_parent = input_elem.getparent()
-                # remove the input element
-                date_parent.remove(input_elem)
-                # try to remove last modified p
-                last_modified_p = date_parent.xpath('./p')
-                if last_modified_p and len(last_modified_p) > 0:
-                    date_parent.remove(last_modified_p[0])
-            # rebuild the html raw string
-            raw = etree.tostring(root, encoding='unicode')
-        except Exception as e:
-            self.log('preprocess_raw_html error -> {}'.format(e))
-        return raw