diff --git a/recipes/guardian.recipe b/recipes/guardian.recipe index 1944a4b7e20e..870d9c811a13 100644 --- a/recipes/guardian.recipe +++ b/recipes/guardian.recipe @@ -10,8 +10,6 @@ from calibre import random_user_agent from calibre.web.feeds.news import BasicNewsRecipe from datetime import date -from lxml import etree - def classes(classes): q = frozenset(classes.split(' ')) @@ -50,17 +48,21 @@ class Guardian(BasicNewsRecipe): dict(attrs={'class': lambda x: x and 'submeta' in x.split()}), dict(attrs={'data-component': ['share', 'social']}), dict(attrs={'data-link-name': 'block share'}), + dict(attrs={'data-print-layout': 'hide'}), + dict(id=['dfp-ad--survey', 'sub-nav-root', 'the-caption', 'bannerandheader']), + {'for': 'the-checkbox'}, + dict(href='#maincontent'), + dict(role='navigation'), dict(attrs={'class': lambda x: x and 'inline-expand-image' in x}), dict(name='a', attrs={'aria-label': lambda x: x and 'Share On' in x}), dict(name='a', attrs={'class': lambda x: x and 'social__action js-social__action--top' in x}), dict(name='div', attrs={'id': 'share-count-root'}), dict(attrs={'class': lambda x: x and 'modern-visible' in x.split()}), classes('badge-slot reveal-caption__checkbox mobile-only element-rich-link'), - dict(name=['link', 'meta', 'style', 'svg', 'input', 'source']), + dict(name=['link', 'meta', 'style', 'svg', 'input', 'source', 'noscript', 'button']), ] remove_tags_after = [ - dict(attrs={'class': lambda x: x and 'content__article-body' in x.split()}), - dict(attrs={'class': lambda x: x and 'article-body-commercial-selector' in x.split()}), + classes('content__article-body js-bottom-marker article-body-commercial-selector'), ] extra_css = """ @@ -117,55 +119,3 @@ class Guardian(BasicNewsRecipe): if date.today().weekday() in (5, 6): feeds += self.parse_section('https://www.theguardian.com/theguardian/weekend', 'Weekend - ') return feeds - - def preprocess_html(self, soup): - old_body = soup.find('body') - if old_body is not None: - main_column = soup.find(**classes('content__main-column')) - if main_column is None: - for section in soup.findAll('section'): - if section.find('h1') is not None: - main_column = section - break - if main_column is not None: - body = soup.new_tag('body') - body.append(main_column) - old_body.replaceWith(body) - - for img in soup.findAll('img', srcset=True): - img['src'] = img['srcset'].partition(' ')[0] - img['srcset'] = '' - return soup - - def preprocess_raw_html(self, raw, url): - try: - root = self.index_to_soup(raw, as_tree=True) - # find out Share on facebook elements - share_platforms = root.xpath('//div//div//div/ul/li/a[starts-with(@aria-label,"Share on")]') - if share_platforms and len(share_platforms) > 0: - parent_div = share_platforms[0].getparent().getparent().getparent() - ul_elem = share_platforms[0].getparent().getparent() - # remove ul element - parent_div.remove(ul_elem) - - share_counts = root.xpath('//div//div//div[contains(@id,"-count-root")]') - if share_counts and len(share_counts) > 0: - share_parent_div = share_counts[0].getparent().getparent() - # remove share count element - share_parent_div.remove(share_counts[0].getparent()) - - pub_date_div = root.xpath('//div//div//div//input[@id="dateToggle"]') - if pub_date_div: - input_elem = pub_date_div[0] - date_parent = input_elem.getparent() - # remove the input element - date_parent.remove(input_elem) - # try to remove last modified p - last_modified_p = date_parent.xpath('./p') - if last_modified_p and len(last_modified_p) > 0: - date_parent.remove(last_modified_p[0]) - # rebuild the html raw string - raw = etree.tostring(root, encoding='unicode') - except Exception as e: - self.log('preprocess_raw_html error -> {}'.format(e)) - return raw