WIP: Port calibre to python 3

.gitignore

@@ -13,6 +13,8 @@ manual/locale
 build
 dist
 docs
+env*
+.idea
 resources/localization
 resources/scripts.pickle
 resources/ebook-convert-complete.pickle

manual/custom.py

@@ -236,7 +236,7 @@ def render_options(cmd, groups, options_header=True, add_program=True, header_le
 
 
 def mark_options(raw):
-    raw = re.sub(r'(\s+)--(\s+)', ur'\1``--``\2', raw)
+    raw = re.sub(r'(\s+)--(\s+)', r'\1``--``\2', raw)
 
     def sub(m):
         opt = m.group()
@@ -274,7 +274,7 @@ def cli_docs(app):
     info(bold('creating CLI documentation...'))
     documented_cmds, undocumented_cmds = get_cli_docs()
 
-    documented_cmds.sort(cmp=lambda x, y: cmp(x[0], y[0]))
+    documented_cmds.sort(key=lambda x: x[0])
     undocumented_cmds.sort()
 
     documented = [' '*4 + c[0] for c in documented_cmds]

recipes/DrawAndCook.recipe

@@ -1,3 +1,4 @@
+from __future__ import print_function
 from calibre.web.feeds.news import BasicNewsRecipe
 import re
 
@@ -30,7 +31,7 @@ class DrawAndCook(BasicNewsRecipe):
             articles = self.make_links(url)
             if articles:
                 feeds.append((title, articles))
-        print 'feeds are: ', feeds
+        print('feeds are: ', feeds)
         return feeds
 
     def make_links(self, url):
@@ -45,9 +46,9 @@ class DrawAndCook(BasicNewsRecipe):
             'li', attrs={'data-id': re.compile(r'artwork_entry_\d+', re.DOTALL)})
         for recipe in recipes:
             page_url = self.INDEX + recipe.a['href']
-            print 'page_url is: ', page_url
+            print('page_url is: ', page_url)
             title = recipe.find('strong').string
-            print 'title is: ', title
+            print('title is: ', title)
             current_articles.append(
                 {'title': title, 'url': page_url, 'description': '', 'date': date})
         return current_articles

recipes/air_force_times.recipe

@@ -26,17 +26,17 @@ class AirForceTimes(BasicNewsRecipe):
     auto_cleanup = True
 
     feeds = [
-        ('Home','http://feeds.feedburner.com/rss/category/air-home?format=xml'),
-        ('Health Benefits','http://feeds.feedburner.com/rss/category/air-healthbenefits?format=xml'),
-        ('Retirement Benefits','http://feeds.feedburner.com/rss/category/air-retirementbenefits?format=xml'),
-        ('Veterans Benefits','http://feeds.feedburner.com/rss/category/air-VeteransBenefits?format=xml'),
-        ('Education Benefits','http://feeds.feedburner.com/rss/category/air-educationbenefits?format=xml'),
-        ('Adventure','http://feeds.feedburner.com/rss/category/air-adventure?format=xml'),
-        ('Entertainment','http://feeds.feedburner.com/rss/category/air-Entertainment?format=xml'),
-        ('Careers','http://feeds.feedburner.com/rss/category/air-careers?format=xml'),
-        ('Technology','http://feeds.feedburner.com/rss/category/air-technology?format=xml'),
-        ('Opinion','http://feeds.feedburner.com/rss/category/air-opinion?format=xml'),
-        ('Pay','http://feeds.feedburner.com/rss/category/air-pay?format=xml'),
-        ('Guard','http://feeds.feedburner.com/rss/category/air-guard?format=xml'),
-        ('Your Air Force','http://feeds.feedburner.com/rss/category/air-yourairforce?format=xml'),
+        ('Home', 'http://feeds.feedburner.com/rss/category/air-home?format=xml'),
+        ('Health Benefits', 'http://feeds.feedburner.com/rss/category/air-healthbenefits?format=xml'),
+        ('Retirement Benefits', 'http://feeds.feedburner.com/rss/category/air-retirementbenefits?format=xml'),
+        ('Veterans Benefits', 'http://feeds.feedburner.com/rss/category/air-VeteransBenefits?format=xml'),
+        ('Education Benefits', 'http://feeds.feedburner.com/rss/category/air-educationbenefits?format=xml'),
+        ('Adventure', 'http://feeds.feedburner.com/rss/category/air-adventure?format=xml'),
+        ('Entertainment', 'http://feeds.feedburner.com/rss/category/air-Entertainment?format=xml'),
+        ('Careers', 'http://feeds.feedburner.com/rss/category/air-careers?format=xml'),
+        ('Technology', 'http://feeds.feedburner.com/rss/category/air-technology?format=xml'),
+        ('Opinion', 'http://feeds.feedburner.com/rss/category/air-opinion?format=xml'),
+        ('Pay', 'http://feeds.feedburner.com/rss/category/air-pay?format=xml'),
+        ('Guard', 'http://feeds.feedburner.com/rss/category/air-guard?format=xml'),
+        ('Your Air Force', 'http://feeds.feedburner.com/rss/category/air-yourairforce?format=xml'),
     ]

recipes/al_monitor.recipe

@@ -1,5 +1,6 @@
 #!/usr/bin/env  python2
 # -*- coding: utf-8 -*-
+from __future__ import print_function
 __license__ = 'GPL v3'
 __copyright__ = '2014, spswerling'
 '''
@@ -210,4 +211,4 @@ class AlMonitor(BasicNewsRecipe):
         curframe = inspect.currentframe()
         calframe = inspect.getouterframes(curframe, 2)
         calname = calframe[1][3].upper()
-        print('[' + calname + '] ' + msg[0:100])
+        print(('[' + calname + '] ' + msg[0:100]))

recipes/alejakomiksu_com.recipe

@@ -21,7 +21,7 @@ class AlejaKomiksu(BasicNewsRecipe):
     remove_attributes = ['style', 'font']
     ignore_duplicate_articles = {'title', 'url'}
 
-    keep_only_tags = dict(attrs={'class': ['akNews__header','akNews__body']})
+    keep_only_tags = dict(attrs={'class': ['akNews__header', 'akNews__body']})
 
     feeds = [(u'Wiadomości', 'http://www.alejakomiksu.com/rss.php5')]
 

recipes/am730.recipe

@@ -1,5 +1,6 @@
 # vim:fileencoding=UTF-8
 from __future__ import unicode_literals
+from __future__ import print_function
 __license__ = 'GPL v3'
 __copyright__ = '2013, Eddie Lau'
 __Date__ = ''
@@ -60,7 +61,7 @@ class AM730(BasicNewsRecipe):
             title = href.split('/')[-1].split('-')[0]
             title = urllib.unquote(title.encode('ASCII'))  # .decode('utf-8')
             if self.debug:
-                print title
+                print(title)
             try:
                 if articles.index({'title':title,'url':href})>=0:
                     # print 'already added'
@@ -73,7 +74,7 @@ class AM730(BasicNewsRecipe):
             if (len(articles) >= self.max_articles_per_feed):
                 break
         if self.debug:
-            print articles
+            print(articles)
         return (sectionName,articles)
 
     def parse_index(self):
@@ -89,8 +90,8 @@ class AM730(BasicNewsRecipe):
         SectionsArticles=[]
         for (title, url) in Sections:
             if self.debug:
-                print title
-                print url
+                print(title)
+                print(url)
             SectionsArticles.append(self.getAMSectionArticles(title,url))
 #        feeds.append(articles[0]['url'])
         return SectionsArticles

recipes/ambito.recipe

@@ -37,22 +37,22 @@ class Ambito(BasicNewsRecipe):
 
     keep_only_tags    = [
                           dict(name='h6', attrs={'class': lambda x: x and 'bajada' in x.split()})
-                          ,dict(name='span', attrs={'class': lambda x: x and 'dia' in x.split()})
-                          ,dict(attrs={'class': lambda x: x and 'titulo-noticia' in x.split()})
-                          ,dict(attrs={'class': lambda x: x and 'foto-perfil-columnista' in x.split()})
-                          ,dict(attrs={'class': lambda x: x and 'despliegue-noticia' in x.split()})
+                          , dict(name='span', attrs={'class': lambda x: x and 'dia' in x.split()})
+                          , dict(attrs={'class': lambda x: x and 'titulo-noticia' in x.split()})
+                          , dict(attrs={'class': lambda x: x and 'foto-perfil-columnista' in x.split()})
+                          , dict(attrs={'class': lambda x: x and 'despliegue-noticia' in x.split()})
                         ]
-    remove_tags       = [dict(name=['object','link','embed','iframe','meta','link'])]
+    remove_tags       = [dict(name=['object', 'link', 'embed', 'iframe', 'meta', 'link'])]
 
     feeds = [
              (u'Principales Noticias', u'http://www.ambito.com/rss/noticiasp.asp')
-             ,(u'Economia'            , u'http://www.ambito.com/rss/noticias.asp?S=Econom%EDa')
-             ,(u'Politica'            , u'http://www.ambito.com/rss/noticias.asp?S=Pol%EDtica')
-             ,(u'Informacion General' , u'http://www.ambito.com/rss/noticias.asp?S=Informaci%F3n%20General')
-             ,(u'Campo'               , u'http://www.ambito.com/rss/noticias.asp?S=Agro')
-             ,(u'Internacionales'     , u'http://www.ambito.com/rss/noticias.asp?S=Internacionales')
-             ,(u'Deportes'            , u'http://www.ambito.com/rss/noticias.asp?S=Deportes')
-             ,(u'Espectaculos'        , u'http://www.ambito.com/rss/noticias.asp?S=Espect%E1culos')
-             ,(u'Tecnologia'          , u'http://www.ambito.com/rss/noticias.asp?S=Tecnolog%EDa')
-             ,(u'Ambito Nacional'     , u'http://www.ambito.com/rss/noticias.asp?S=Ambito%20Nacional')
+             , (u'Economia', u'http://www.ambito.com/rss/noticias.asp?S=Econom%EDa')
+             , (u'Politica', u'http://www.ambito.com/rss/noticias.asp?S=Pol%EDtica')
+             , (u'Informacion General', u'http://www.ambito.com/rss/noticias.asp?S=Informaci%F3n%20General')
+             , (u'Campo', u'http://www.ambito.com/rss/noticias.asp?S=Agro')
+             , (u'Internacionales', u'http://www.ambito.com/rss/noticias.asp?S=Internacionales')
+             , (u'Deportes', u'http://www.ambito.com/rss/noticias.asp?S=Deportes')
+             , (u'Espectaculos', u'http://www.ambito.com/rss/noticias.asp?S=Espect%E1culos')
+             , (u'Tecnologia', u'http://www.ambito.com/rss/noticias.asp?S=Tecnolog%EDa')
+             , (u'Ambito Nacional', u'http://www.ambito.com/rss/noticias.asp?S=Ambito%20Nacional')
             ]

recipes/ambito_financiero.recipe

@@ -9,11 +9,12 @@ ambito.com/diario
 '''
 
 import time
-import urllib
+import six.moves.urllib.request, six.moves.urllib.parse, six.moves.urllib.error
 import re
 from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
+import six
 
 
 class Ambito_Financiero(BasicNewsRecipe):
@@ -58,7 +59,7 @@ class Ambito_Financiero(BasicNewsRecipe):
         br = BasicNewsRecipe.get_browser(self)
         br.open(self.INDEX)
         if self.username is not None and self.password is not None:
-            data = urllib.urlencode({
+            data = six.moves.urllib.parse.urlencode({
                 'txtUser': self.username,
                 'txtPassword': self.password
             })
@@ -98,7 +99,7 @@ class Ambito_Financiero(BasicNewsRecipe):
         if self.session_id:
             l, s, r = url.rpartition('/')
             artid, s1, r1 = r.partition('-')
-            data = urllib.urlencode({'id': artid, 'id_session': self.session_id})
+            data = six.moves.urllib.parse.urlencode({'id': artid, 'id_session': self.session_id})
             response = self.browser.open(
                 'http://data.ambito.com/diario/cuerpo_noticia.asp', data
             )
@@ -109,12 +110,12 @@ class Ambito_Financiero(BasicNewsRecipe):
                 cfind = smallsoup.find('div', id="contenido_data")
                 if cfind:
                     p.append(cfind)
-                return unicode(soup)
+                return six.text_type(soup)
         return raw_html
 
     def cleanup(self):
         if self.session_id is not None:
-            data = urllib.urlencode({'session_id': self.session_id})
+            data = six.moves.urllib.parse.urlencode({'session_id': self.session_id})
             self.browser.open(
                 'http://www.ambito.com/diario/no-cache/login/x_logout.asp', data
             )

recipes/american_thinker.recipe

@@ -7,6 +7,7 @@ import html5lib
 from calibre.web.feeds.news import BasicNewsRecipe
 from calibre.utils.cleantext import clean_xml_chars
 from lxml import etree
+import six
 
 
 class AmericanThinker(BasicNewsRecipe):
@@ -34,7 +35,7 @@ class AmericanThinker(BasicNewsRecipe):
             namespaceHTMLElements=False)
         for x in root.xpath('''descendant-or-self::*[@class and contains(concat(' ', normalize-space(@class), ' '), ' article_body ') and (@class and contains(concat(' ', normalize-space(@class), ' '), ' bottom '))]'''):  # noqa
             x.getparent().remove(x)
-        return etree.tostring(root, encoding=unicode)
+        return etree.tostring(root, encoding=six.text_type)
 
     feeds = [(u'http://feeds.feedburner.com/americanthinker'),
              (u'http://feeds.feedburner.com/AmericanThinkerBlog')

recipes/android_com_pl.recipe

@@ -15,5 +15,5 @@ class Android_com_pl(BasicNewsRecipe):
     remove_tags_after = [{'class': 'post-content'}]
     remove_tags = [dict(name='ul', attrs={'class': 'tags small-tags'}), dict(name='a', attrs={'onclick': 'return ss_plugin_loadpopup_js(this);'})]
     preprocess_regexps = [
-        (re.compile(ur'<p>.{,1}</p>', re.DOTALL), lambda match: '')]
+        (re.compile(u'<p>.{,1}</p>', re.DOTALL), lambda match: '')]
     feeds = [(u'Android', u'http://android.com.pl/feed/')]

recipes/apple_daily.recipe

@@ -1,5 +1,7 @@
 # vim:fileencoding=UTF-8
 from __future__ import unicode_literals
+import six
+from six.moves import range
 __license__ = 'GPL v3'
 __copyright__ = '2013-2015, Eddie Lau'
 __Date__ = ''
@@ -162,7 +164,7 @@ class AppleDaily(BasicNewsRecipe):
                     article_titles.append(force_unicode(a.title, 'utf-8'))
 
         mi.comments = self.description
-        if not isinstance(mi.comments, unicode):
+        if not isinstance(mi.comments, six.text_type):
             mi.comments = mi.comments.decode('utf-8', 'replace')
         mi.comments += ('\n\n' + _('Articles in this issue: ') + '\n' +
                         '\n\n'.join(article_titles))
@@ -256,24 +258,24 @@ class AppleDaily(BasicNewsRecipe):
                     if os.path.exists(last):
                         with open(last, 'rb') as fi:
                             src = fi.read().decode('utf-8')
-                        src = src.replace('height:260px !important;','')  # fix flow-player div tag parent
+                        src = src.replace('height:260px !important;', '')  # fix flow-player div tag parent
                         soup = BeautifulSoup(src)
                         body = soup.find('body')
                         if body is not None:
-                            prefix = '/'.join('..'for i in range(2 *
-                                                                 len(re.findall(r'link\d+', last))))
+                            prefix = '/'.join('..'for i in list(range(2 *
+                                                                 len(re.findall(r'link\d+', last)))))
                             templ = self.navbar.generate(True, num, j, len(f),
                                                          not self.has_single_feed,
                                                          a.orig_url, __appname__, prefix=prefix,
                                                          center=self.center_navbar)
                             translatedTempl =re.sub(
-                                '<hr.*<br','<hr>本篇由 '+__appname__+
+                                '<hr.*<br', '<hr>本篇由 '+__appname__+
                                 ' 快取自 <a href="http://hkm.appledaily.com/" >蘋果日報</a> ; <a href="'+a.orig_url+'">本篇來源位置</a>。'+
-                                '<br',templ.render(doctype='xhtml').decode('utf-8'),flags=re.S)
+                                '<br', templ.render(doctype='xhtml').decode('utf-8'), flags=re.S)
                             elem = BeautifulSoup(translatedTempl).find('div')
                             body.insert(len(body.contents), elem)
                             with open(last, 'wb') as fi:
-                                fi.write(unicode(soup).encode('utf-8'))
+                                fi.write(six.text_type(soup).encode('utf-8'))
         if len(feeds) == 0:
             raise Exception('All feeds are empty, aborting.')
 

recipes/appledaily_tw.recipe

@@ -104,7 +104,7 @@ class AppledailyTW(BasicNewsRecipe):
     ]
 
     def preprocess_raw_html(self, raw_html, url):
-        raw_html = re.sub(ur'<a href=".*?<br><br>.*?<\/a>', '', raw_html)
+        raw_html = re.sub(r'<a href=".*?<br><br>.*?<\/a>', '', raw_html)
         raw_html = re.sub(
-            ur'<title>(.*?)[\s]+\|.*<\/title>', '<title>\1<\/title>', raw_html)
+            r'<title>(.*?)[\s]+\|.*<\/title>', '<title>\1<\/title>', raw_html)
         return raw_html

recipes/auto_prove.recipe

@@ -1,4 +1,5 @@
 #!/usr/bin/env  python2
+from __future__ import print_function
 __license__ = 'GPL v3'
 __author__ = 'GabrieleMarini, based on Darko Miletic'
 __copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>, Gabriele Marini'
@@ -56,7 +57,7 @@ class AutoPR(BasicNewsRecipe):
         ]:
             soup = self.index_to_soup(url)
             soup = soup.find('channel')
-            print soup
+            print(soup)
 
             for article in soup.findAllNext('item'):
                 title = self.tag_to_string(article.title)

recipes/azstarnet.recipe

@@ -4,7 +4,7 @@ __copyright__ = '2009-2010, Darko Miletic <darko.miletic at gmail.com>'
 '''
 azstarnet.com
 '''
-import urllib
+import six.moves.urllib.request, six.moves.urllib.parse, six.moves.urllib.error
 from calibre.web.feeds.news import BasicNewsRecipe
 
 
@@ -31,7 +31,7 @@ class Azstarnet(BasicNewsRecipe):
         br = BasicNewsRecipe.get_browser(self)
         br.open('http://azstarnet.com/')
         if self.username is not None and self.password is not None:
-            data = urllib.urlencode({'m': 'login', 'u': self.username, 'p': self.password, 'z': 'http://azstarnet.com/'
+            data = six.moves.urllib.parse.urlencode({'m': 'login', 'u': self.username, 'p': self.password, 'z': 'http://azstarnet.com/'
                                      })
             br.open('http://azstarnet.com/app/registration/proxy.php', data)
         return br

recipes/banat_news.recipe

@@ -52,12 +52,12 @@ class BanatNews(BasicNewsRecipe):
 
     feeds = [
 
-    ('Balita'      		, 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=101'),
-    ('Opinyon'       		, 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=102'),
-    ('Kalingawan'     		, 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=104'),
-    ('Showbiz'          		, 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=62'),
-    ('Palaro'            		, 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=103'),
-    ('Imong Kapalaran'        	, 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=105')
+    ('Balita', 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=101'),
+    ('Opinyon', 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=102'),
+    ('Kalingawan', 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=104'),
+    ('Showbiz', 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=62'),
+    ('Palaro', 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=103'),
+    ('Imong Kapalaran', 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=105')
     ]
 
 # process the printer friendly version of article

recipes/barrons.recipe

@@ -7,7 +7,7 @@ from __future__ import (unicode_literals, division, absolute_import,
 
 import json
 from mechanize import Request
-from urllib import quote
+from six.moves.urllib.parse import quote
 
 from calibre.web.feeds.news import BasicNewsRecipe
 

recipes/bash_org_pl.recipe

@@ -1,4 +1,5 @@
 from calibre.web.feeds.news import BasicNewsRecipe
+from six.moves import range
 
 
 class Bash_org_pl(BasicNewsRecipe):

recipes/benchmark_pl.recipe

@@ -16,8 +16,8 @@ class BenchmarkPl(BasicNewsRecipe):
     extra_css = 'ul {list-style-type: none;}'
     no_stylesheets = True
     use_embedded_content = False
-    preprocess_regexps = [(re.compile(ur'<h3><span style="font-size: small;">&nbsp;Zobacz poprzednie <a href="http://www.benchmark.pl/news/zestawienie/grupa_id/135">Opinie dnia:</a></span>.*</body>',  # noqa
-        re.DOTALL | re.IGNORECASE), lambda match: '</body>'), (re.compile(ur'Więcej o .*?</ul>', re.DOTALL | re.IGNORECASE), lambda match: '')]  # noqa
+    preprocess_regexps = [(re.compile(u'<h3><span style="font-size: small;">&nbsp;Zobacz poprzednie <a href="http://www.benchmark.pl/news/zestawienie/grupa_id/135">Opinie dnia:</a></span>.*</body>',  # noqa
+        re.DOTALL | re.IGNORECASE), lambda match: '</body>'), (re.compile(u'Więcej o .*?</ul>', re.DOTALL | re.IGNORECASE), lambda match: '')]  # noqa
 
     keep_only_tags = [dict(id=['articleHeader', 'articleGallery']), dict(
         name='div', attrs={'class': ['m_zwykly', 'gallery']}), dict(id='article')]