diff --git a/recipes/chicago_tribune.recipe b/recipes/chicago_tribune.recipe index b005c1f67505..9989128d8577 100644 --- a/recipes/chicago_tribune.recipe +++ b/recipes/chicago_tribune.recipe @@ -6,92 +6,48 @@ __docformat__ = 'restructuredtext en' from calibre.web.feeds.news import BasicNewsRecipe +def classes(classes): + q = frozenset(classes.split(' ')) + return dict(attrs={ + 'class': lambda x: x and frozenset(x.split()).intersection(q)}) + + class ChicagoTribune(BasicNewsRecipe): title = 'Chicago Tribune' - __author__ = 'Kovid Goyal, Sujata Raman and a.peter' + __author__ = 'Kovid Goyal' description = 'Politics, local and business news from Chicago' language = 'en' - version = 2 - use_embedded_content = False no_stylesheets = True remove_javascript = True - recursions = 1 - - keep_only_tags = [dict(name='div', attrs={'class': ["story", "entry-asset asset hentry"]}), - dict(name='div', attrs={ - 'id': ["pagebody", "story", "maincontentcontainer"]}), - ] - remove_tags_after = [{'class': ['photo_article', ]}] - remove_empty_feeds = True - match_regexps = [r'page=[0-9]+'] + keep_only_tags = [ + dict(name='h1'), + dict(attrs={'data-content-size': 'leadart'}), + dict(itemprop='articleBody'), + ] remove_tags = [ - {'id': ["moduleArticleTools", "content-bottom", "rail", "articleRelates module", "toolSet", - "relatedrailcontent", "div-wrapper", "beta", "atp-comments", "footer", 'gallery-subcontent', 'subFooter']}, - {'class': ["clearfix", "relatedTitle", "articleRelates module", - "asset-footer", "tools", "comments", "featurePromo", "featurePromo fp-topjobs brownBackground", 'ndn_embed', - "clearfix fullSpan brownBackground", "curvedContent", 'nextgen-share-tools', 'outbrainTools', 'google-ad-story-bottom']}, - dict(name='font', attrs={'id': ["cr-other-headlines"]})] - extra_css = ''' - h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;} - h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;} - .byline {font-family:Arial,Helvetica,sans-serif; font-size:xx-small;} - .date {font-family:Arial,Helvetica,sans-serif; font-size:xx-small;} - p{font-family:Arial,Helvetica,sans-serif;font-size:small;} - .copyright {font-family:Arial,Helvetica,sans-serif;font-size:xx-small;text-align:center} - .story{font-family:Arial,Helvetica,sans-serif;font-size:small;} - .entry-asset asset hentry{font-family:Arial,Helvetica,sans-serif;font-size:small;} - .pagebody{font-family:Arial,Helvetica,sans-serif;font-size:small;} - .maincontentcontainer{font-family:Arial,Helvetica,sans-serif;font-size:small;} - .story-body{font-family:Arial,Helvetica,sans-serif;font-size:small;} - body{font-family:Helvetica,Arial,sans-serif;font-size:small;} - ''' + classes('trb_ar_cont trb_ar_main_ad trb_em_r_cc'), + ] + feeds = [ - ('Latest news', 'http://feeds.chicagotribune.com/chicagotribune/news/'), - ('Local news', 'http://feeds.chicagotribune.com/chicagotribune/news/local/'), - ('Nation/world', 'http://feeds.chicagotribune.com/chicagotribune/news/nationworld/'), - ('Hot topics', 'http://feeds.chicagotribune.com/chicagotribune/hottopics/'), - ('Most E-mailed stories', 'http://feeds.chicagotribune.com/chicagotribune/email/'), - ('Opinion', 'http://feeds.chicagotribune.com/chicagotribune/opinion/'), - ('Off Topic', 'http://feeds.chicagotribune.com/chicagotribune/offtopic/'), - ('Politics', 'http://feeds.feedburner.com/chicagotribune/cloutstreet/'), - ('Business news', 'http://feeds.chicagotribune.com/chicagotribune/business/'), - ('Jobs and Careers', 'http://feeds.chicagotribune.com/chicagotribune/career/'), - ('Local scene', 'http://feeds.chicagotribune.com/chicagohomes/localscene/'), - ('Phil Rosenthal', 'http://feeds.chicagotribune.com/chicagotribune/rosenthal/'), - ('Your Money', 'http://feeds.chicagotribune.com/chicagotribune/yourmoney/'), - ('Jon Hilkevitch - Getting around', - 'http://feeds.chicagotribune.com/chicagotribune/gettingaround/'), - ('Jon Yates - What\'s your problem?', - 'http://feeds.chicagotribune.com/chicagotribune/problem/'), - ('Garisson Keillor', 'http://feeds.chicagotribune.com/chicagotribune/keillor/'), - ('Marks Jarvis - On Money', - 'http://feeds.chicagotribune.com/chicagotribune/marksjarvisonmoney/'), - ('Sports', 'http://feeds.chicagotribune.com/chicagotribune/sports/'), - ('Arts and Architecture', - 'http://feeds.chicagotribune.com/chicagotribune/arts/'), - ('Books', 'http://feeds.chicagotribune.com/chicagotribune/books/'), - ('Movies', 'http://feeds.chicagotribune.com/chicagotribune/movies/'), - ('Music', 'http://feeds.chicagotribune.com/chicagotribune/music/'), - ('TV', 'http://feeds.chicagotribune.com/chicagotribune/tv/'), - ('Hypertext', 'http://feeds.chicagotribune.com/chicagotribune/hypertext/'), - ('iPhone Blog', 'http://feeds.feedburner.com/redeye/iphoneblog'), - ('Julie\'s Health Club', - 'http://feeds.chicagotribune.com/chicagotribune_julieshealthclub/'), + ('Breaking news', 'https://www.chicagotribune.com/news/local/breaking/rss2.0.xml'), + ('Trending news', 'https://www.chicagotribune.com/news/trending/rss2.0.xml'), + ('Opinion', 'https://www.chicagotribune.com/news/opinion/rss2.0.xml'), + ('Business news', 'https://www.chicagotribune.com/business/rss2.0.xml'), + ('Sports', 'https://www.chicagotribune.com/sports/rss2.0.xml'), + ('Arts and Entertainment', + 'https://www.chicagotribune.com/entertainment/rss2.0.xml'), + ('Life & Style', + 'https://www.chicagotribune.com/lifestyles/rss2.0.xml'), ] - def get_article_url(self, article): - url = BasicNewsRecipe.get_article_url(self, article) - url = url.split('/')[-2] - encoding = {'0B': '.', '0C': '/', '0A': '0', '0F': '=', '0G': '&', - '0D': '?', '0E': '-', '0N': '.com', '0L': 'http://', '0S': - 'www.', '0I': '_', '0H': ','} - for k, v in encoding.iteritems(): - url = url.replace(k, v) - return url.partition('?')[0] + def preprocess_html(self, soup): + for img in soup.findAll('img', attrs={'data-baseurl': True}): + img['src'] = img['data-baseurl'] + return soup def skip_ad_pages(self, soup): text = soup.find(text='click here to continue to article') @@ -100,19 +56,3 @@ class ChicagoTribune(BasicNewsRecipe): url = a.get('href') if url: return self.index_to_soup(url, raw=True) - - def postprocess_html(self, soup, first_fetch): - # Remove the navigation bar. It was kept until now to be able to follow - # the links to further pages. But now we don't need them anymore. - for nav in soup.findAll(attrs={'class': ['toppaginate', 'article-nav clearfix']}): - nav.extract() - - for t in soup.findAll(['table', 'tr', 'td']): - t.name = 'div' - - for tag in soup.findAll('form', dict(attrs={'name': ["comments_form"]})): - tag.extract() - for tag in soup.findAll('font', dict(attrs={'id': ["cr-other-headlines"]})): - tag.extract() - - return soup