Skip to content

Commit

Permalink
Update Chicago Tribune
Browse files Browse the repository at this point in the history
  • Loading branch information
@kovidgoyal
kovidgoyal committed Jan 17, 2019
1 parent 59daec1 commit b011cdd
Showing 1 changed file with 28 additions and 88 deletions.
116 changes: 28 additions & 88 deletions recipes/chicago_tribune.recipe
View file
Expand Up @@ -6,92 +6,48 @@ __docformat__ = 'restructuredtext en'
from calibre.web.feeds.news import BasicNewsRecipe


def classes(classes):
q = frozenset(classes.split(' '))
return dict(attrs={
'class': lambda x: x and frozenset(x.split()).intersection(q)})


class ChicagoTribune(BasicNewsRecipe):

title = 'Chicago Tribune'
__author__ = 'Kovid Goyal, Sujata Raman and a.peter'
__author__ = 'Kovid Goyal'
description = 'Politics, local and business news from Chicago'
language = 'en'
version = 2

use_embedded_content = False
no_stylesheets = True
remove_javascript = True
recursions = 1

keep_only_tags = [dict(name='div', attrs={'class': ["story", "entry-asset asset hentry"]}),
dict(name='div', attrs={
'id': ["pagebody", "story", "maincontentcontainer"]}),
]
remove_tags_after = [{'class': ['photo_article', ]}]
remove_empty_feeds = True

match_regexps = [r'page=[0-9]+']
keep_only_tags = [
dict(name='h1'),
dict(attrs={'data-content-size': 'leadart'}),
dict(itemprop='articleBody'),
]

remove_tags = [
{'id': ["moduleArticleTools", "content-bottom", "rail", "articleRelates module", "toolSet",
"relatedrailcontent", "div-wrapper", "beta", "atp-comments", "footer", 'gallery-subcontent', 'subFooter']},
{'class': ["clearfix", "relatedTitle", "articleRelates module",
"asset-footer", "tools", "comments", "featurePromo", "featurePromo fp-topjobs brownBackground", 'ndn_embed',
"clearfix fullSpan brownBackground", "curvedContent", 'nextgen-share-tools', 'outbrainTools', 'google-ad-story-bottom']},
dict(name='font', attrs={'id': ["cr-other-headlines"]})]
extra_css = '''
h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
.byline {font-family:Arial,Helvetica,sans-serif; font-size:xx-small;}
.date {font-family:Arial,Helvetica,sans-serif; font-size:xx-small;}
p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
.copyright {font-family:Arial,Helvetica,sans-serif;font-size:xx-small;text-align:center}
.story{font-family:Arial,Helvetica,sans-serif;font-size:small;}
.entry-asset asset hentry{font-family:Arial,Helvetica,sans-serif;font-size:small;}
.pagebody{font-family:Arial,Helvetica,sans-serif;font-size:small;}
.maincontentcontainer{font-family:Arial,Helvetica,sans-serif;font-size:small;}
.story-body{font-family:Arial,Helvetica,sans-serif;font-size:small;}
body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
'''
classes('trb_ar_cont trb_ar_main_ad trb_em_r_cc'),
]

feeds = [
('Latest news', 'http://feeds.chicagotribune.com/chicagotribune/news/'),
('Local news', 'http://feeds.chicagotribune.com/chicagotribune/news/local/'),
('Nation/world', 'http://feeds.chicagotribune.com/chicagotribune/news/nationworld/'),
('Hot topics', 'http://feeds.chicagotribune.com/chicagotribune/hottopics/'),
('Most E-mailed stories', 'http://feeds.chicagotribune.com/chicagotribune/email/'),
('Opinion', 'http://feeds.chicagotribune.com/chicagotribune/opinion/'),
('Off Topic', 'http://feeds.chicagotribune.com/chicagotribune/offtopic/'),
('Politics', 'http://feeds.feedburner.com/chicagotribune/cloutstreet/'),
('Business news', 'http://feeds.chicagotribune.com/chicagotribune/business/'),
('Jobs and Careers', 'http://feeds.chicagotribune.com/chicagotribune/career/'),
('Local scene', 'http://feeds.chicagotribune.com/chicagohomes/localscene/'),
('Phil Rosenthal', 'http://feeds.chicagotribune.com/chicagotribune/rosenthal/'),
('Your Money', 'http://feeds.chicagotribune.com/chicagotribune/yourmoney/'),
('Jon Hilkevitch - Getting around',
'http://feeds.chicagotribune.com/chicagotribune/gettingaround/'),
('Jon Yates - What\'s your problem?',
'http://feeds.chicagotribune.com/chicagotribune/problem/'),
('Garisson Keillor', 'http://feeds.chicagotribune.com/chicagotribune/keillor/'),
('Marks Jarvis - On Money',
'http://feeds.chicagotribune.com/chicagotribune/marksjarvisonmoney/'),
('Sports', 'http://feeds.chicagotribune.com/chicagotribune/sports/'),
('Arts and Architecture',
'http://feeds.chicagotribune.com/chicagotribune/arts/'),
('Books', 'http://feeds.chicagotribune.com/chicagotribune/books/'),
('Movies', 'http://feeds.chicagotribune.com/chicagotribune/movies/'),
('Music', 'http://feeds.chicagotribune.com/chicagotribune/music/'),
('TV', 'http://feeds.chicagotribune.com/chicagotribune/tv/'),
('Hypertext', 'http://feeds.chicagotribune.com/chicagotribune/hypertext/'),
('iPhone Blog', 'http://feeds.feedburner.com/redeye/iphoneblog'),
('Julie\'s Health Club',
'http://feeds.chicagotribune.com/chicagotribune_julieshealthclub/'),
('Breaking news', 'https://www.chicagotribune.com/news/local/breaking/rss2.0.xml'),
('Trending news', 'https://www.chicagotribune.com/news/trending/rss2.0.xml'),
('Opinion', 'https://www.chicagotribune.com/news/opinion/rss2.0.xml'),
('Business news', 'https://www.chicagotribune.com/business/rss2.0.xml'),
('Sports', 'https://www.chicagotribune.com/sports/rss2.0.xml'),
('Arts and Entertainment',
'https://www.chicagotribune.com/entertainment/rss2.0.xml'),
('Life & Style',
'https://www.chicagotribune.com/lifestyles/rss2.0.xml'),
]

def get_article_url(self, article):
url = BasicNewsRecipe.get_article_url(self, article)
url = url.split('/')[-2]
encoding = {'0B': '.', '0C': '/', '0A': '0', '0F': '=', '0G': '&',
'0D': '?', '0E': '-', '0N': '.com', '0L': 'http://', '0S':
'www.', '0I': '_', '0H': ','}
for k, v in encoding.iteritems():
url = url.replace(k, v)
return url.partition('?')[0]
def preprocess_html(self, soup):
for img in soup.findAll('img', attrs={'data-baseurl': True}):
img['src'] = img['data-baseurl']
return soup

def skip_ad_pages(self, soup):
text = soup.find(text='click here to continue to article')
Expand All @@ -100,19 +56,3 @@ class ChicagoTribune(BasicNewsRecipe):
url = a.get('href')
if url:
return self.index_to_soup(url, raw=True)

def postprocess_html(self, soup, first_fetch):
# Remove the navigation bar. It was kept until now to be able to follow
# the links to further pages. But now we don't need them anymore.
for nav in soup.findAll(attrs={'class': ['toppaginate', 'article-nav clearfix']}):
nav.extract()

for t in soup.findAll(['table', 'tr', 'td']):
t.name = 'div'

for tag in soup.findAll('form', dict(attrs={'name': ["comments_form"]})):
tag.extract()
for tag in soup.findAll('font', dict(attrs={'id': ["cr-other-headlines"]})):
tag.extract()

return soup

0 comments on commit b011cdd

Please sign in to comment.