From 71995d5e1bec6f34ef836e1dc7416e27dd3cfe30 Mon Sep 17 00:00:00 2001 From: Dirk Makowski Date: Fri, 11 Jan 2013 02:57:43 +0100 Subject: [PATCH] Port pelican to python 3. Stays compatible with 2.x series, thanks to an unified codebase. --- .gitignore | 2 + .travis.yml | 4 +- dev_requirements.txt | 7 +- docs/changelog.rst | 2 +- docs/conf.py | 21 ++-- pelican/__init__.py | 14 ++- pelican/contents.py | 45 ++++--- pelican/generators.py | 36 +++--- pelican/log.py | 36 +++--- pelican/paginator.py | 5 +- pelican/plugins/assets.py | 1 + pelican/plugins/github_activity.py | 1 + pelican/plugins/gzip_cache.py | 2 +- pelican/plugins/html_rst_directive.py | 5 +- pelican/plugins/initialized.py | 4 +- pelican/plugins/related_posts.py | 4 +- pelican/plugins/sitemap.py | 15 ++- pelican/readers.py | 10 +- pelican/rstdirectives.py | 4 +- pelican/server.py | 20 ++++ pelican/settings.py | 16 +-- pelican/signals.py | 2 + pelican/tools/pelican_import.py | 51 +++++--- pelican/tools/pelican_quickstart.py | 82 +++++++------ pelican/tools/pelican_themes.py | 17 +-- pelican/tools/templates/Makefile.in | 2 +- pelican/tools/templates/develop_server.sh.in | 12 +- pelican/utils.py | 120 ++++++++++++++++--- pelican/writers.py | 19 +-- samples/pelican.conf.py | 10 +- setup.py | 4 +- tests/default_conf.py | 7 +- tests/support.py | 6 +- tests/test_contents.py | 6 +- tests/test_generators.py | 43 +++---- tests/test_importer.py | 23 +--- tests/test_pelican.py | 6 +- tests/test_plugins.py | 2 +- tests/test_readers.py | 39 ++---- tests/test_settings.py | 4 +- tests/test_utils.py | 7 +- tests/test_webassets.py | 1 + tox.ini | 65 +++++++++- 43 files changed, 495 insertions(+), 287 deletions(-) create mode 100644 pelican/server.py diff --git a/.gitignore b/.gitignore index 9f9404ef82..1ae0e9f6c5 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ tags .tox .coverage htmlcov +six-*.egg/ +*.orig diff --git a/.travis.yml b/.travis.yml index bb9a22e4b2..e92b0abc34 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,11 +2,13 @@ language: python python: - "2.6" - "2.7" +# - "3.2" before_install: - sudo apt-get update -qq - sudo apt-get install -qq ruby-sass install: - - pip install nose unittest2 mock --use-mirrors + - pip install nose mock --use-mirrors + - if [[ $TRAVIS_PYTHON_VERSION == '3.2' ]]; then pip install --use-mirrors unittest2py3k; else pip install --use-mirrors unittest2; fi - pip install . --use-mirrors - pip install Markdown - pip install webassets diff --git a/dev_requirements.txt b/dev_requirements.txt index acf017738b..e0189d35ff 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,8 +1,9 @@ # Tests -unittest2 mock + # Optional Packages Markdown -BeautifulSoup +BeautifulSoup4 +lxml typogrify -webassets \ No newline at end of file +webassets diff --git a/docs/changelog.rst b/docs/changelog.rst index e29c94c0d3..aa18ea9e9c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,7 @@ Release history 3.2 (XXXX-XX-XX) ================ -* [...] +* Support for Python 3! 3.1 (2012-12-04) ================ diff --git a/docs/conf.py b/docs/conf.py index 2a11fe3e8a..6c8ececa94 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals import sys, os sys.path.append(os.path.abspath('..')) @@ -10,8 +11,8 @@ extensions = ['sphinx.ext.autodoc',] source_suffix = '.rst' master_doc = 'index' -project = u'Pelican' -copyright = u'2010, Alexis Metaireau and contributors' +project = 'Pelican' +copyright = '2010, Alexis Metaireau and contributors' exclude_patterns = ['_build'] version = __version__ release = __major__ @@ -34,16 +35,16 @@ # -- Options for LaTeX output -------------------------------------------------- latex_documents = [ - ('index', 'Pelican.tex', u'Pelican Documentation', - u'Alexis Métaireau', 'manual'), + ('index', 'Pelican.tex', 'Pelican Documentation', + 'Alexis Métaireau', 'manual'), ] # -- Options for manual page output -------------------------------------------- man_pages = [ - ('index', 'pelican', u'pelican documentation', - [u'Alexis Métaireau'], 1), - ('pelican-themes', 'pelican-themes', u'A theme manager for Pelican', - [u'Mickaël Raybaud'], 1), - ('themes', 'pelican-theming', u'How to create themes for Pelican', - [u'The Pelican contributors'], 1) + ('index', 'pelican', 'pelican documentation', + ['Alexis Métaireau'], 1), + ('pelican-themes', 'pelican-themes', 'A theme manager for Pelican', + ['Mickaël Raybaud'], 1), + ('themes', 'pelican-theming', 'How to create themes for Pelican', + ['The Pelican contributors'], 1) ] diff --git a/pelican/__init__.py b/pelican/__init__.py index e2ea7f76cf..566cb3b02e 100644 --- a/pelican/__init__.py +++ b/pelican/__init__.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function +import six + import os import re import sys @@ -55,7 +59,7 @@ def init_plugins(self): self.plugins = self.settings['PLUGINS'] for plugin in self.plugins: # if it's a string, then import it - if isinstance(plugin, basestring): + if isinstance(plugin, six.string_types): logger.debug("Loading plugin `{0}' ...".format(plugin)) plugin = __import__(plugin, globals(), locals(), 'module') @@ -265,7 +269,7 @@ def get_instance(args): settings = read_settings(args.settings, override=get_config(args)) cls = settings.get('PELICAN_CLASS') - if isinstance(cls, basestring): + if isinstance(cls, six.string_types): module, cls_name = cls.rsplit('.', 1) module = __import__(module) cls = getattr(module, cls_name) @@ -311,15 +315,15 @@ def main(): "Nothing to generate.") files_found_error = False time.sleep(1) # sleep to avoid cpu load - except Exception, e: + except Exception as e: logger.warning( "Caught exception \"{}\". Reloading.".format(e) ) continue else: pelican.run() - except Exception, e: - logger.critical(unicode(e)) + except Exception as e: + logger.critical(e) if (args.verbosity == logging.DEBUG): raise diff --git a/pelican/contents.py b/pelican/contents.py index d675a2adc8..4655d4ccd0 100644 --- a/pelican/contents.py +++ b/pelican/contents.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function +import six + import copy import locale import logging @@ -11,8 +14,10 @@ from pelican.settings import _DEFAULT_CONFIG -from pelican.utils import slugify, truncate_html_words, memoized +from pelican.utils import (slugify, truncate_html_words, memoized, + python_2_unicode_compatible) from pelican import signals +import pelican.utils logger = logging.getLogger(__name__) @@ -85,13 +90,8 @@ def __init__(self, content, metadata=None, settings=None, self.date_format = self.date_format[1] if hasattr(self, 'date'): - encoded_date = self.date.strftime( - self.date_format.encode('ascii', 'xmlcharrefreplace')) - - if platform == 'win32': - self.locale_date = encoded_date.decode(stdin.encoding) - else: - self.locale_date = encoded_date.decode('utf') + self.locale_date = pelican.utils.strftime(self.date, + self.date_format) # manage status if not hasattr(self, 'status'): @@ -167,7 +167,7 @@ def replacer(m): origin = '/'.join((siteurl, self._context['filenames'][value].url)) else: - logger.warning(u"Unable to find {fn}, skipping url" + logger.warning("Unable to find {fn}, skipping url" " replacement".format(fn=value)) return m.group('markup') + m.group('quote') + origin \ @@ -243,10 +243,10 @@ class Article(Page): class Quote(Page): base_properties = ('author', 'date') - +@python_2_unicode_compatible class URLWrapper(object): def __init__(self, name, settings): - self.name = unicode(name) + self.name = name self.slug = slugify(self.name) self.settings = settings @@ -257,12 +257,9 @@ def __hash__(self): return hash(self.name) def __eq__(self, other): - return self.name == unicode(other) + return self.name == other def __str__(self): - return str(self.name.encode('utf-8', 'replace')) - - def __unicode__(self): return self.name def _from_settings(self, key, get_page_name=False): @@ -272,14 +269,14 @@ def _from_settings(self, key, get_page_name=False): Useful for pagination.""" setting = "%s_%s" % (self.__class__.__name__.upper(), key) value = self.settings[setting] - if not isinstance(value, basestring): - logger.warning(u'%s is set to %s' % (setting, value)) + if not isinstance(value, six.string_types): + logger.warning('%s is set to %s' % (setting, value)) return value else: if get_page_name: - return unicode(os.path.splitext(value)[0]).format(**self.as_dict()) + return os.path.splitext(value)[0].format(**self.as_dict()) else: - return unicode(value).format(**self.as_dict()) + return value.format(**self.as_dict()) page_name = property(functools.partial(_from_settings, key='URL', get_page_name=True)) url = property(functools.partial(_from_settings, key='URL')) @@ -292,13 +289,14 @@ class Category(URLWrapper): class Tag(URLWrapper): def __init__(self, name, *args, **kwargs): - super(Tag, self).__init__(unicode.strip(name), *args, **kwargs) + super(Tag, self).__init__(name.strip(), *args, **kwargs) class Author(URLWrapper): pass +@python_2_unicode_compatible class StaticContent(object): def __init__(self, src, dst=None, settings=None): if not settings: @@ -309,9 +307,6 @@ def __init__(self, src, dst=None, settings=None): self.save_as = os.path.join(settings['OUTPUT_PATH'], self.url) def __str__(self): - return str(self.filepath.encode('utf-8', 'replace')) - - def __unicode__(self): return self.filepath @@ -319,7 +314,7 @@ def is_valid_content(content, f): try: content.check_properties() return True - except NameError, e: - logger.error(u"Skipping %s: impossible to find informations about" + except NameError as e: + logger.error("Skipping %s: impossible to find informations about" "'%s'" % (f, e)) return False diff --git a/pelican/generators.py b/pelican/generators.py index b5c1b94492..0d2fa6a9d3 100644 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function + import os import math import random import logging import datetime -import subprocess import shutil from codecs import open @@ -119,7 +120,7 @@ def _update_context(self, items): for item in items: value = getattr(self, item) if hasattr(value, 'items'): - value = value.items() + value = list(value.items()) self.context[item] = value @@ -133,8 +134,8 @@ def get_source(self, environment, template): if template != self.path or not os.path.exists(self.fullpath): raise TemplateNotFound(template) mtime = os.path.getmtime(self.fullpath) - with file(self.fullpath) as f: - source = f.read().decode('utf-8') + with open(self.fullpath, 'r', encoding='utf-8') as f: + source = f.read() return source, self.fullpath, \ lambda: mtime == os.path.getmtime(self.fullpath) @@ -323,8 +324,8 @@ def generate_context(self): try: signals.article_generate_preread.send(self) content, metadata = read_file(f, settings=self.settings) - except Exception, e: - logger.warning(u'Could not process %s\n%s' % (f, str(e))) + except Exception as e: + logger.warning('Could not process %s\n%s' % (f, str(e))) continue # if no category is set, use the name of the path as a category @@ -333,8 +334,7 @@ def generate_context(self): if (self.settings['USE_FOLDER_AS_CATEGORY'] and os.path.dirname(f) != article_path): # if the article is in a subdirectory - category = os.path.basename(os.path.dirname(f))\ - .decode('utf-8') + category = os.path.basename(os.path.dirname(f)) else: # if the article is not in a subdirectory category = self.settings['DEFAULT_CATEGORY'] @@ -366,8 +366,8 @@ def generate_context(self): elif article.status == "draft": self.drafts.append(article) else: - logger.warning(u"Unknown status %s for file %s, skipping it." % - (repr(unicode.encode(article.status, 'utf-8')), + logger.warning("Unknown status %s for file %s, skipping it." % + (repr(article.status), repr(f))) self.articles, self.translations = process_translations(all_articles) @@ -394,7 +394,7 @@ def generate_context(self): tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True) tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')] - tags = map(itemgetter(1), tag_cloud) + tags = list(map(itemgetter(1), tag_cloud)) if tags: max_count = max(tags) steps = self.settings.get('TAG_CLOUD_STEPS') @@ -450,8 +450,8 @@ def generate_context(self): exclude=self.settings['PAGE_EXCLUDES']): try: content, metadata = read_file(f, settings=self.settings) - except Exception, e: - logger.warning(u'Could not process %s\n%s' % (f, str(e))) + except Exception as e: + logger.warning('Could not process %s\n%s' % (f, str(e))) continue signals.pages_generate_context.send(self, metadata=metadata) page = Page(content, metadata, settings=self.settings, @@ -466,8 +466,8 @@ def generate_context(self): elif page.status == "hidden": hidden_pages.append(page) else: - logger.warning(u"Unknown status %s for file %s, skipping it." % - (repr(unicode.encode(page.status, 'utf-8')), + logger.warning("Unknown status %s for file %s, skipping it." % + (repr(page.status), repr(f))) self.pages, self.translations = process_translations(all_pages) @@ -550,7 +550,7 @@ def _create_pdf(self, obj, output_path): # print "Generating pdf for", obj.filename, " in ", output_pdf with open(obj.filename) as f: self.pdfcreator.createPdf(text=f.read(), output=output_pdf) - logger.info(u' [ok] writing %s' % output_pdf) + logger.info(' [ok] writing %s' % output_pdf) def generate_context(self): pass @@ -558,7 +558,7 @@ def generate_context(self): def generate_output(self, writer=None): # we don't use the writer passed as argument here # since we write our own files - logger.info(u' Generating PDF files...') + logger.info(' Generating PDF files...') pdf_path = os.path.join(self.output_path, 'pdf') if not os.path.exists(pdf_path): try: @@ -583,6 +583,6 @@ def _create_source(self, obj, output_path): copy('', obj.filename, dest) def generate_output(self, writer=None): - logger.info(u' Generating source files...') + logger.info(' Generating source files...') for object in chain(self.context['articles'], self.context['pages']): self._create_source(object, self.output_path) diff --git a/pelican/log.py b/pelican/log.py index 9590d7f659..2ce11ea206 100644 --- a/pelican/log.py +++ b/pelican/log.py @@ -1,3 +1,6 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function + __all__ = [ 'init' ] @@ -9,7 +12,7 @@ from logging import Formatter, getLogger, StreamHandler, DEBUG -RESET_TERM = u'\033[0;m' +RESET_TERM = '\033[0;m' COLOR_CODES = { 'red': 31, @@ -24,37 +27,38 @@ def ansi(color, text): """Wrap text in an ansi escape sequence""" code = COLOR_CODES[color] - return u'\033[1;{0}m{1}{2}'.format(code, text, RESET_TERM) + return '\033[1;{0}m{1}{2}'.format(code, text, RESET_TERM) class ANSIFormatter(Formatter): """ - Convert a `logging.LogReport' object into colored text, using ANSI escape sequences. + Convert a `logging.LogRecord' object into colored text, using ANSI escape sequences. """ ## colors: def format(self, record): - if record.levelname is 'INFO': - return ansi('cyan', '-> ') + unicode(record.msg) - elif record.levelname is 'WARNING': - return ansi('yellow', record.levelname) + ': ' + unicode(record.msg) - elif record.levelname is 'ERROR': - return ansi('red', record.levelname) + ': ' + unicode(record.msg) - elif record.levelname is 'CRITICAL': - return ansi('bgred', record.levelname) + ': ' + unicode(record.msg) - elif record.levelname is 'DEBUG': - return ansi('bggrey', record.levelname) + ': ' + unicode(record.msg) + msg = str(record.msg) + if record.levelname == 'INFO': + return ansi('cyan', '-> ') + msg + elif record.levelname == 'WARNING': + return ansi('yellow', record.levelname) + ': ' + msg + elif record.levelname == 'ERROR': + return ansi('red', record.levelname) + ': ' + msg + elif record.levelname == 'CRITICAL': + return ansi('bgred', record.levelname) + ': ' + msg + elif record.levelname == 'DEBUG': + return ansi('bggrey', record.levelname) + ': ' + msg else: - return ansi('white', record.levelname) + ': ' + unicode(record.msg) + return ansi('white', record.levelname) + ': ' + msg class TextFormatter(Formatter): """ - Convert a `logging.LogReport' object into text. + Convert a `logging.LogRecord' object into text. """ def format(self, record): - if not record.levelname or record.levelname is 'INFO': + if not record.levelname or record.levelname == 'INFO': return record.msg else: return record.levelname + ': ' + record.msg diff --git a/pelican/paginator.py b/pelican/paginator.py index fe871491c4..067215c28c 100644 --- a/pelican/paginator.py +++ b/pelican/paginator.py @@ -1,3 +1,6 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function + # From django.core.paginator from math import ceil @@ -37,7 +40,7 @@ def _get_page_range(self): Returns a 1-based range of pages for iterating through within a template for loop. """ - return range(1, self.num_pages + 1) + return list(range(1, self.num_pages + 1)) page_range = property(_get_page_range) diff --git a/pelican/plugins/assets.py b/pelican/plugins/assets.py index b5d1cf76c5..9c819a4b6b 100644 --- a/pelican/plugins/assets.py +++ b/pelican/plugins/assets.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals """ Asset management plugin for Pelican =================================== diff --git a/pelican/plugins/github_activity.py b/pelican/plugins/github_activity.py index f2ba1da751..cc139efc9c 100644 --- a/pelican/plugins/github_activity.py +++ b/pelican/plugins/github_activity.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function """ Copyright (c) Marco Milanesi diff --git a/pelican/plugins/gzip_cache.py b/pelican/plugins/gzip_cache.py index 784a6ca0ce..9080629e4a 100644 --- a/pelican/plugins/gzip_cache.py +++ b/pelican/plugins/gzip_cache.py @@ -68,7 +68,7 @@ def create_gzip_file(filepath): logger.debug('Compressing: %s' % filepath) compressed = gzip.open(compressed_path, 'wb') compressed.writelines(uncompressed) - except Exception, ex: + except Exception as ex: logger.critical('Gzip compression failed: %s' % ex) finally: compressed.close() diff --git a/pelican/plugins/html_rst_directive.py b/pelican/plugins/html_rst_directive.py index d0a656f575..2b2bceb381 100644 --- a/pelican/plugins/html_rst_directive.py +++ b/pelican/plugins/html_rst_directive.py @@ -1,6 +1,7 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals from docutils import nodes from docutils.parsers.rst import directives, Directive -from pelican import log """ HTML tags for reStructuredText @@ -52,7 +53,7 @@ class RawHtml(Directive): has_content = True def run(self): - html = u' '.join(self.content) + html = ' '.join(self.content) node = nodes.raw('', html, format='html') return [node] diff --git a/pelican/plugins/initialized.py b/pelican/plugins/initialized.py index 5e4cf17442..b4b5ab76a6 100644 --- a/pelican/plugins/initialized.py +++ b/pelican/plugins/initialized.py @@ -1,7 +1,9 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function from pelican import signals def test(sender): - print "%s initialized !!" % sender + print("%s initialized !!" % sender) def register(): signals.initialized.connect(test) diff --git a/pelican/plugins/related_posts.py b/pelican/plugins/related_posts.py index 6771502342..c2765c3c0e 100644 --- a/pelican/plugins/related_posts.py +++ b/pelican/plugins/related_posts.py @@ -41,8 +41,8 @@ def add_related_posts(generator, metadata): if len(related_posts) < 1: return - relation_score = dict(zip(set(related_posts), map(related_posts.count, - set(related_posts)))) + relation_score = dict(list(zip(set(related_posts), list(map(related_posts.count, + set(related_posts)))))) ranked_related = sorted(relation_score, key=relation_score.get) metadata["related_posts"] = ranked_related[:5] diff --git a/pelican/plugins/sitemap.py b/pelican/plugins/sitemap.py index ebce1f04b9..0ade12885e 100644 --- a/pelican/plugins/sitemap.py +++ b/pelican/plugins/sitemap.py @@ -1,3 +1,6 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + import collections import os.path @@ -7,19 +10,19 @@ from pelican import signals, contents -TXT_HEADER = u"""{0}/index.html +TXT_HEADER = """{0}/index.html {0}/archives.html {0}/tags.html {0}/categories.html """ -XML_HEADER = u""" +XML_HEADER = """ """ -XML_URL = u""" +XML_URL = """ {0}/{1} {2} @@ -28,7 +31,7 @@ """ -XML_FOOTER = u""" +XML_FOOTER = """ """ @@ -86,7 +89,7 @@ def __init__(self, context, settings, path, theme, output_path, *null): 'yearly', 'never') if isinstance(pris, dict): - for k, v in pris.iteritems(): + for k, v in pris.items(): if k in valid_keys and not isinstance(v, (int, float)): default = self.priorities[k] warning("sitemap plugin: priorities must be numbers") @@ -99,7 +102,7 @@ def __init__(self, context, settings, path, theme, output_path, *null): warning("sitemap plugin: using the default values") if isinstance(chfreqs, dict): - for k, v in chfreqs.iteritems(): + for k, v in chfreqs.items(): if k in valid_keys and v not in valid_chfreqs: default = self.changefreqs[k] warning("sitemap plugin: invalid changefreq `{0}'".format(v)) diff --git a/pelican/readers.py b/pelican/readers.py index 6bd2822d2e..b182b6fa01 100644 --- a/pelican/readers.py +++ b/pelican/readers.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function +import six + import os import re try: @@ -20,15 +23,16 @@ asciidoc = True except ImportError: asciidoc = False +import re from pelican.contents import Category, Tag, Author from pelican.utils import get_date, pelican_open _METADATA_PROCESSORS = { - 'tags': lambda x, y: [Tag(tag, y) for tag in unicode(x).split(',')], + 'tags': lambda x, y: [Tag(tag, y) for tag in x.split(',')], 'date': lambda x, y: get_date(x), - 'status': lambda x, y: unicode.strip(x), + 'status': lambda x, y: x.strip(), 'category': Category, 'author': Author, } @@ -242,7 +246,7 @@ def read_file(filename, fmt=None, settings=None): if filename_metadata: match = re.match(filename_metadata, base) if match: - for k, v in match.groupdict().iteritems(): + for k, v in match.groupdict().items(): if k not in metadata: k = k.lower() # metadata must be lowercase metadata[k] = reader.process_metadata(k, v) diff --git a/pelican/rstdirectives.py b/pelican/rstdirectives.py index c677144dab..fb4a6c93a5 100644 --- a/pelican/rstdirectives.py +++ b/pelican/rstdirectives.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function + from docutils import nodes, utils from docutils.parsers.rst import directives, roles, Directive from pygments.formatters import HtmlFormatter @@ -32,7 +34,7 @@ def run(self): # take an arbitrary option if more than one is given formatter = self.options and VARIANTS[self.options.keys()[0]] \ or DEFAULT - parsed = highlight(u'\n'.join(self.content), lexer, formatter) + parsed = highlight('\n'.join(self.content), lexer, formatter) return [nodes.raw('', parsed, format='html')] directives.register_directive('code-block', Pygments) diff --git a/pelican/server.py b/pelican/server.py new file mode 100644 index 0000000000..0f5a71d269 --- /dev/null +++ b/pelican/server.py @@ -0,0 +1,20 @@ +from __future__ import print_function +try: + import SimpleHTTPServer as srvmod +except ImportError: + import http.server as srvmod + +try: + import SocketServer as socketserver +except ImportError: + import socketserver + +PORT = 8000 + +Handler = srvmod.SimpleHTTPRequestHandler + +httpd = socketserver.TCPServer(("", PORT), Handler) + +print("serving at port", PORT) +httpd.serve_forever() + diff --git a/pelican/settings.py b/pelican/settings.py index 692fc9832f..0748f99152 100644 --- a/pelican/settings.py +++ b/pelican/settings.py @@ -1,11 +1,13 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function +import six + import copy import imp import inspect import os import locale import logging -import re from os.path import isabs @@ -54,8 +56,8 @@ 'CATEGORY_SAVE_AS': 'category/{slug}.html', 'TAG_URL': 'tag/{slug}.html', 'TAG_SAVE_AS': 'tag/{slug}.html', - 'AUTHOR_URL': u'author/{slug}.html', - 'AUTHOR_SAVE_AS': u'author/{slug}.html', + 'AUTHOR_URL': 'author/{slug}.html', + 'AUTHOR_SAVE_AS': 'author/{slug}.html', 'RELATIVE_URLS': True, 'DEFAULT_LANG': 'en', 'TAG_CLOUD_STEPS': 4, @@ -146,7 +148,7 @@ def configure_settings(settings): # if locales is not a list, make it one locales = settings['LOCALE'] - if isinstance(locales, basestring): + if isinstance(locales, six.string_types): locales = [locales] # try to set the different locales, fallback on the default. @@ -155,7 +157,7 @@ def configure_settings(settings): for locale_ in locales: try: - locale.setlocale(locale.LC_ALL, locale_) + locale.setlocale(locale.LC_ALL, str(locale_)) break # break if it is successful except locale.Error: pass @@ -200,14 +202,14 @@ def configure_settings(settings): "of the Webassets plugin") if 'OUTPUT_SOURCES_EXTENSION' in settings: - if not isinstance(settings['OUTPUT_SOURCES_EXTENSION'], str): + if not isinstance(settings['OUTPUT_SOURCES_EXTENSION'], six.string_types): settings['OUTPUT_SOURCES_EXTENSION'] = _DEFAULT_CONFIG['OUTPUT_SOURCES_EXTENSION'] logger.warn("Detected misconfiguration with OUTPUT_SOURCES_EXTENSION." " falling back to the default extension " + _DEFAULT_CONFIG['OUTPUT_SOURCES_EXTENSION']) filename_metadata = settings.get('FILENAME_METADATA') - if filename_metadata and not isinstance(filename_metadata, basestring): + if filename_metadata and not isinstance(filename_metadata, six.string_types): logger.error("Detected misconfiguration with FILENAME_METADATA" " setting (must be string or compiled pattern), falling" "back to the default") diff --git a/pelican/signals.py b/pelican/signals.py index d592599ac3..1367a11753 100644 --- a/pelican/signals.py +++ b/pelican/signals.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function from blinker import signal initialized = signal('pelican_initialized') diff --git a/pelican/tools/pelican_import.py b/pelican/tools/pelican_import.py index fdd1ac51dc..b1a2865f81 100755 --- a/pelican/tools/pelican_import.py +++ b/pelican/tools/pelican_import.py @@ -1,7 +1,12 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function import argparse -from HTMLParser import HTMLParser +try: + from html.parser import HTMLParser +except ImportError: + from HTMLParser import HTMLParser import os import subprocess import sys @@ -15,14 +20,14 @@ def wp2fields(xml): """Opens a wordpress XML file, and yield pelican fields""" try: - from BeautifulSoup import BeautifulStoneSoup + from bs4 import BeautifulSoup except ImportError: error = ('Missing dependency ' - '"BeautifulSoup" required to import Wordpress XML files.') + '"BeautifulSoup4" and "lxml" required to import Wordpress XML files.') sys.exit(error) xmlfile = open(xml, encoding='utf-8').read() - soup = BeautifulStoneSoup(xmlfile) + soup = BeautifulSoup(xmlfile, "xml") items = soup.rss.channel.findAll('item') for item in items: @@ -54,10 +59,10 @@ def wp2fields(xml): def dc2fields(file): """Opens a Dotclear export file, and yield pelican fields""" try: - from BeautifulSoup import BeautifulStoneSoup + from bs4 import BeautifulSoup except ImportError: error = ('Missing dependency ' - '"BeautifulSoup" required to import Dotclear files.') + '"BeautifulSoup4" and "lxml" required to import Dotclear files.') sys.exit(error) @@ -142,13 +147,27 @@ def dc2fields(file): if len(tag) > 1: if int(tag[:1]) == 1: newtag = tag.split('"')[1] - tags.append(unicode(BeautifulStoneSoup(newtag,convertEntities=BeautifulStoneSoup.HTML_ENTITIES ))) + tags.append( + BeautifulSoup( + newtag + , "xml" + ) + # bs4 always outputs UTF-8 + .decode('utf-8') + ) else: i=1 j=1 while(i <= int(tag[:1])): newtag = tag.split('"')[j].replace('\\','') - tags.append(unicode(BeautifulStoneSoup(newtag,convertEntities=BeautifulStoneSoup.HTML_ENTITIES ))) + tags.append( + BeautifulSoup( + newtag + , "xml" + ) + # bs4 always outputs UTF-8 + .decode('utf-8') + ) i=i+1 if j < int(tag[:1])*2: j=j+2 @@ -244,7 +263,7 @@ def fields2pelican(fields, out_markup, output_path, dircat=False, strip_raw=Fals # Replace newlines with paragraphs wrapped with

so # HTML is valid before conversion paragraphs = content.splitlines() - paragraphs = [u'

{0}

'.format(p) for p in paragraphs] + paragraphs = ['

{0}

'.format(p) for p in paragraphs] new_content = ''.join(paragraphs) fp.write(new_content) @@ -264,7 +283,7 @@ def fields2pelican(fields, out_markup, output_path, dircat=False, strip_raw=Fals elif rc > 0: error = "Please, check your Pandoc installation." exit(error) - except OSError, e: + except OSError as e: error = "Pandoc execution failed: %s" % e exit(error) @@ -284,7 +303,7 @@ def fields2pelican(fields, out_markup, output_path, dircat=False, strip_raw=Fals def main(): parser = argparse.ArgumentParser( description="Transform feed, Wordpress or Dotclear files to reST (rst) " - "or Markdown (md) files. Be sure to have pandoc installed.", + "or Markdown (md) files. Be sure to have pandoc installed", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument(dest='input', help='The input file to read') @@ -304,10 +323,10 @@ def main(): help="Strip raw HTML code that can't be converted to " "markup such as flash embeds or iframes (wordpress import only)") parser.add_argument('--disable-slugs', action='store_true', - dest='disable_slugs', - help='Disable storing slugs from imported posts within output. ' - 'With this disabled, your Pelican URLs may not be consistent ' - 'with your original posts.') + dest='disable_slugs', + help='Disable storing slugs from imported posts within output. ' + 'With this disabled, your Pelican URLs may not be consistent ' + 'with your original posts.') args = parser.parse_args() @@ -339,4 +358,4 @@ def main(): fields2pelican(fields, args.markup, args.output, dircat=args.dircat or False, strip_raw=args.strip_raw or False, - disable_slugs=args.disable_slugs or False) + strip_slugs=args.disable_slugs or False) diff --git a/pelican/tools/pelican_quickstart.py b/pelican/tools/pelican_quickstart.py index 447a80c541..9f25f2fe23 100755 --- a/pelican/tools/pelican_quickstart.py +++ b/pelican/tools/pelican_quickstart.py @@ -1,5 +1,8 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # + +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function +import six import os import string @@ -29,11 +32,22 @@ 'lang': 'en' } +def _input_compat(prompt): + if six.PY3: + r = input(prompt) + else: + r = raw_input(prompt).decode('utf-8') + return r + +if six.PY3: + str_compat = str +else: + str_compat = unicode def decoding_strings(f): def wrapper(*args, **kwargs): out = f(*args, **kwargs) - if isinstance(out, basestring): + if isinstance(out, six.string_types): # todo: make encoding configurable? return out.decode(sys.stdin.encoding) return out @@ -55,14 +69,14 @@ def get_template(name, as_encoding='utf-8'): @decoding_strings -def ask(question, answer=str, default=None, l=None): - if answer == str: +def ask(question, answer=str_compat, default=None, l=None): + if answer == str_compat: r = '' while True: if default: - r = raw_input('> {0} [{1}] '.format(question, default)) + r = _input_compat('> {0} [{1}] '.format(question, default)) else: - r = raw_input('> {0} '.format(question, default)) + r = _input_compat('> {0} '.format(question, default)) r = r.strip() @@ -84,11 +98,11 @@ def ask(question, answer=str, default=None, l=None): r = None while True: if default is True: - r = raw_input('> {0} (Y/n) '.format(question)) + r = _input_compat('> {0} (Y/n) '.format(question)) elif default is False: - r = raw_input('> {0} (y/N) '.format(question)) + r = _input_compat('> {0} (y/N) '.format(question)) else: - r = raw_input('> {0} (y/n) '.format(question)) + r = _input_compat('> {0} (y/n) '.format(question)) r = r.strip().lower() @@ -108,9 +122,9 @@ def ask(question, answer=str, default=None, l=None): r = None while True: if default: - r = raw_input('> {0} [{1}] '.format(question, default)) + r = _input_compat('> {0} [{1}] '.format(question, default)) else: - r = raw_input('> {0} '.format(question)) + r = _input_compat('> {0} '.format(question)) r = r.strip() @@ -125,7 +139,7 @@ def ask(question, answer=str, default=None, l=None): print('You must enter an integer') return r else: - raise NotImplemented('Argument `answer` must be str, bool, or integer') + raise NotImplemented('Argument `answer` must be str_compat, bool, or integer') def main(): @@ -158,14 +172,14 @@ def main(): print('Using project associated with current virtual environment.' 'Will save to:\n%s\n' % CONF['basedir']) else: - CONF['basedir'] = os.path.abspath(ask('Where do you want to create your new web site?', answer=str, default=args.path)) + CONF['basedir'] = os.path.abspath(ask('Where do you want to create your new web site?', answer=str_compat, default=args.path)) - CONF['sitename'] = ask('What will be the title of this web site?', answer=str, default=args.title) - CONF['author'] = ask('Who will be the author of this web site?', answer=str, default=args.author) - CONF['lang'] = ask('What will be the default language of this web site?', str, args.lang or CONF['lang'], 2) + CONF['sitename'] = ask('What will be the title of this web site?', answer=str_compat, default=args.title) + CONF['author'] = ask('Who will be the author of this web site?', answer=str_compat, default=args.author) + CONF['lang'] = ask('What will be the default language of this web site?', str_compat, args.lang or CONF['lang'], 2) if ask('Do you want to specify a URL prefix? e.g., http://example.com ', answer=bool, default=True): - CONF['siteurl'] = ask('What is your URL prefix? (see above example; no trailing slash)', str, CONF['siteurl']) + CONF['siteurl'] = ask('What is your URL prefix? (see above example; no trailing slash)', str_compat, CONF['siteurl']) CONF['with_pagination'] = ask('Do you want to enable article pagination?', bool, bool(CONF['default_pagination'])) @@ -179,38 +193,38 @@ def main(): if mkfile: if ask('Do you want to upload your website using FTP?', answer=bool, default=False): - CONF['ftp_host'] = ask('What is the hostname of your FTP server?', str, CONF['ftp_host']) - CONF['ftp_user'] = ask('What is your username on that server?', str, CONF['ftp_user']) - CONF['ftp_target_dir'] = ask('Where do you want to put your web site on that server?', str, CONF['ftp_target_dir']) + CONF['ftp_host'] = ask('What is the hostname of your FTP server?', str_compat, CONF['ftp_host']) + CONF['ftp_user'] = ask('What is your username on that server?', str_compat, CONF['ftp_user']) + CONF['ftp_target_dir'] = ask('Where do you want to put your web site on that server?', str_compat, CONF['ftp_target_dir']) if ask('Do you want to upload your website using SSH?', answer=bool, default=False): - CONF['ssh_host'] = ask('What is the hostname of your SSH server?', str, CONF['ssh_host']) + CONF['ssh_host'] = ask('What is the hostname of your SSH server?', str_compat, CONF['ssh_host']) CONF['ssh_port'] = ask('What is the port of your SSH server?', int, CONF['ssh_port']) - CONF['ssh_user'] = ask('What is your username on that server?', str, CONF['ssh_user']) - CONF['ssh_target_dir'] = ask('Where do you want to put your web site on that server?', str, CONF['ssh_target_dir']) + CONF['ssh_user'] = ask('What is your username on that server?', str_compat, CONF['ssh_user']) + CONF['ssh_target_dir'] = ask('Where do you want to put your web site on that server?', str_compat, CONF['ssh_target_dir']) if ask('Do you want to upload your website using Dropbox?', answer=bool, default=False): - CONF['dropbox_dir'] = ask('Where is your Dropbox directory?', str, CONF['dropbox_dir']) + CONF['dropbox_dir'] = ask('Where is your Dropbox directory?', str_compat, CONF['dropbox_dir']) try: os.makedirs(os.path.join(CONF['basedir'], 'content')) - except OSError, e: + except OSError as e: print('Error: {0}'.format(e)) try: os.makedirs(os.path.join(CONF['basedir'], 'output')) - except OSError, e: + except OSError as e: print('Error: {0}'.format(e)) try: with codecs.open(os.path.join(CONF['basedir'], 'pelicanconf.py'), 'w', 'utf-8') as fd: conf_python = dict() - for key, value in CONF.iteritems(): + for key, value in CONF.items(): conf_python[key] = repr(value) for line in get_template('pelicanconf.py'): template = string.Template(line) fd.write(template.safe_substitute(conf_python)) fd.close() - except OSError, e: + except OSError as e: print('Error: {0}'.format(e)) try: @@ -219,7 +233,7 @@ def main(): template = string.Template(line) fd.write(template.safe_substitute(CONF)) fd.close() - except OSError, e: + except OSError as e: print('Error: {0}'.format(e)) if mkfile: @@ -229,13 +243,13 @@ def main(): template = string.Template(line) fd.write(template.safe_substitute(CONF)) fd.close() - except OSError, e: + except OSError as e: print('Error: {0}'.format(e)) if develop: conf_shell = dict() - for key, value in CONF.iteritems(): - if isinstance(value, basestring) and ' ' in value: + for key, value in CONF.items(): + if isinstance(value, six.string_types) and ' ' in value: value = '"' + value.replace('"', '\\"') + '"' conf_shell[key] = value try: @@ -244,8 +258,8 @@ def main(): template = string.Template(line) fd.write(template.safe_substitute(conf_shell)) fd.close() - os.chmod((os.path.join(CONF['basedir'], 'develop_server.sh')), 0755) - except OSError, e: + os.chmod((os.path.join(CONF['basedir'], 'develop_server.sh')), 493) # mode 0o755 + except OSError as e: print('Error: {0}'.format(e)) print('Done. Your new project is available at %s' % CONF['basedir']) diff --git a/pelican/tools/pelican_themes.py b/pelican/tools/pelican_themes.py index d13e60e970..8d71535d58 100755 --- a/pelican/tools/pelican_themes.py +++ b/pelican/tools/pelican_themes.py @@ -1,5 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function + +import six import argparse import os @@ -28,7 +31,7 @@ def err(msg, die=None): """Print an error message and exits if an exit code is given""" - sys.stderr.write(str(msg) + '\n') + sys.stderr.write(msg + '\n') if die: sys.exit((die if type(die) is int else 1)) @@ -186,13 +189,13 @@ def install(path, v=False, u=False): for root, dirs, files in os.walk(theme_path): for d in dirs: dname = os.path.join(root, d) - os.chmod(dname, 0755) + os.chmod(dname, 493) # 0o755 for f in files: fname = os.path.join(root, f) - os.chmod(fname, 0644) - except OSError, e: + os.chmod(fname, 420) # 0o644 + except OSError as e: err("Cannot change permissions of files or directory in `{r}':\n{e}".format(r=theme_path, e=str(e)), die=False) - except Exception, e: + except Exception as e: err("Cannot copy `{p}' to `{t}':\n{e}".format(p=path, t=theme_path, e=str(e))) @@ -212,7 +215,7 @@ def symlink(path, v=False): print("Linking `{p}' to `{t}' ...".format(p=path, t=theme_path)) try: os.symlink(path, theme_path) - except Exception, e: + except Exception as e: err("Cannot link `{p}' to `{t}':\n{e}".format(p=path, t=theme_path, e=str(e))) @@ -233,7 +236,7 @@ def clean(v=False): print('Removing {0}'.format(path)) try: os.remove(path) - except OSError, e: + except OSError as e: print('Error: cannot remove {0}'.format(path)) else: c+=1 diff --git a/pelican/tools/templates/Makefile.in b/pelican/tools/templates/Makefile.in index 8e5c80f9a8..8dd9163085 100644 --- a/pelican/tools/templates/Makefile.in +++ b/pelican/tools/templates/Makefile.in @@ -49,7 +49,7 @@ regenerate: clean $$(PELICAN) -r $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS) serve: - cd $$(OUTPUTDIR) && python -m SimpleHTTPServer + cd $$(OUTPUTDIR) && python -m pelican.server devserver: $$(BASEDIR)/develop_server.sh restart diff --git a/pelican/tools/templates/develop_server.sh.in b/pelican/tools/templates/develop_server.sh.in index 41764728fc..1eda47c4b9 100755 --- a/pelican/tools/templates/develop_server.sh.in +++ b/pelican/tools/templates/develop_server.sh.in @@ -20,7 +20,7 @@ PELICAN_PID=$$BASEDIR/pelican.pid function usage(){ echo "usage: $$0 (stop) (start) (restart)" echo "This starts pelican in debug and reload mode and then launches" - echo "A SimpleHTTP server to help site development. It doesn't read" + echo "A pelican.server to help site development. It doesn't read" echo "your pelican options so you edit any paths in your Makefile" echo "you will need to edit it as well" exit 3 @@ -31,14 +31,14 @@ function shut_down(){ PID=$$(cat $$SRV_PID) PROCESS=$$(ps -p $$PID | tail -n 1 | awk '{print $$4}') if [[ $$PROCESS != "" ]]; then - echo "Killing SimpleHTTPServer" + echo "Killing pelican.server" kill $$PID else echo "Stale PID, deleting" fi rm $$SRV_PID else - echo "SimpleHTTPServer PIDFile not found" + echo "pelican.server PIDFile not found" fi if [[ -f $$PELICAN_PID ]]; then @@ -57,15 +57,15 @@ function shut_down(){ } function start_up(){ - echo "Starting up Pelican and SimpleHTTPServer" + echo "Starting up Pelican and pelican.server" shift $$PELICAN --debug --autoreload -r $$INPUTDIR -o $$OUTPUTDIR -s $$CONFFILE $$PELICANOPTS & echo $$! > $$PELICAN_PID cd $$OUTPUTDIR - python -m SimpleHTTPServer & + python -m pelican.server & echo $$! > $$SRV_PID cd $$BASEDIR - sleep 1 && echo 'Pelican and SimpleHTTPServer processes now running in background.' + sleep 1 && echo 'Pelican and pelican.server processes now running in background.' } ### diff --git a/pelican/utils.py b/pelican/utils.py index 6ca797c44c..3a41d04e2a 100644 --- a/pelican/utils.py +++ b/pelican/utils.py @@ -1,10 +1,14 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function +import six + import os import re import pytz import shutil import logging import errno +import locale from collections import defaultdict, Hashable from functools import partial @@ -17,6 +21,77 @@ logger = logging.getLogger(__name__) +def strftime(date, date_format): + """ + Replacement for the builtin strftime(). + + This :func:`strftime()` is compatible to Python 2 and 3. In both cases, + input and output is always unicode. + + Still, Python 3's :func:`strftime()` seems to somehow "normalize" unicode + chars in the format string. So if e.g. your format string contains 'ø' or + 'ä', the result will be 'o' and 'a'. + + See here for an `extensive testcase `_. + + :param date: Any object that sports a :meth:`strftime()` method. + :param date_format: Format string, can always be unicode. + :returns: Unicode string with formatted date. + """ + # As tehkonst confirmed, above mentioned testcase runs correctly on + # Python 2 and 3 on Windows as well. Thanks. + if six.PY3: + # It could be so easy... *sigh* + return date.strftime(date_format) + # TODO Perhaps we should refactor again, so that the + # xmlcharrefreplace-regex-dance is always done, regardless + # of the Python version. + else: + # We must ensure that the format string is an encoded byte + # string, ASCII only WTF!!! + # But with "xmlcharrefreplace" our formatted date will produce + # *yuck* like this: + # "Øl trinken beim Besäufnis" + # --> "Øl trinken beim Besäufnis" + date_format = date_format.encode('ascii', + errors="xmlcharrefreplace") + result = date.strftime(date_format) + # strftime() returns an encoded byte string + # which we must decode into unicode. + lang_code, enc = locale.getlocale(locale.LC_ALL) + if enc: + result = result.decode(enc) + else: + result = unicode(result) + # Convert XML character references back to unicode characters. + if "&#" in result: + result = re.sub(r'&#(?P\d+);' + , lambda m: unichr(int(m.group('num'))) + , result + ) + return result + + + +#---------------------------------------------------------------------------- +# Stolen from Django: django.utils.encoding +# + +def python_2_unicode_compatible(klass): + """ + A decorator that defines __unicode__ and __str__ methods under Python 2. + Under Python 3 it does nothing. + + To support Python 2 and 3 with a single code base, define a __str__ method + returning text and apply this decorator to the class. + """ + if not six.PY3: + klass.__unicode__ = klass.__str__ + klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + return klass + +#---------------------------------------------------------------------------- + class NoFilesError(Exception): pass @@ -78,14 +153,24 @@ def slugify(value): Took from django sources. """ + # TODO Maybe steal again from current Django 1.5dev value = Markup(value).striptags() - if type(value) == unicode: - import unicodedata - from unidecode import unidecode - value = unicode(unidecode(value)) - value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore') - value = unicode(re.sub('[^\w\s-]', '', value).strip().lower()) - return re.sub('[-\s]+', '-', value) + # value must be unicode per se + import unicodedata + from unidecode import unidecode + # unidecode returns str in Py2 and 3, so in Py2 we have to make + # it unicode again + value = unidecode(value) + if isinstance(value, six.binary_type): + value = value.decode('ascii') + # still unicode + value = unicodedata.normalize('NFKD', value) + value = re.sub('[^\w\s-]', '', value).strip().lower() + value = re.sub('[-\s]+', '-', value) + # we want only ASCII chars + value = value.encode('ascii', 'ignore') + # but Pelican should generally use only unicode + return value.decode('ascii') def copy(path, source, destination, destination_path=None, overwrite=False): @@ -137,7 +222,7 @@ def clean_output_dir(path): if not os.path.isdir(path): try: os.remove(path) - except Exception, e: + except Exception as e: logger.error("Unable to delete file %s; %e" % path, e) return @@ -148,13 +233,13 @@ def clean_output_dir(path): try: shutil.rmtree(file) logger.debug("Deleted directory %s" % file) - except Exception, e: + except Exception as e: logger.error("Unable to delete directory %s; %e" % file, e) elif os.path.isfile(file) or os.path.islink(file): try: os.remove(file) logger.debug("Deleted file/link %s" % file) - except Exception, e: + except Exception as e: logger.error("Unable to delete file %s; %e" % file, e) else: logger.error("Unable to delete %s, file type unknown" % file) @@ -180,7 +265,7 @@ def truncate_html_words(s, num, end_text='...'): """ length = int(num) if length <= 0: - return u'' + return '' html4_singlets = ('br', 'col', 'link', 'base', 'img', 'param', 'area', 'hr', 'input') @@ -254,10 +339,10 @@ def process_translations(content_list): for slug, items in grouped_by_slugs: items = list(items) # find items with default language - default_lang_items = filter(attrgetter('in_default_lang'), items) + default_lang_items = list(filter(attrgetter('in_default_lang'), items)) len_ = len(default_lang_items) if len_ > 1: - logger.warning(u'there are %s variants of "%s"' % (len_, slug)) + logger.warning('there are %s variants of "%s"' % (len_, slug)) for x in default_lang_items: logger.warning(' %s' % x.filename) elif len_ == 0: @@ -269,12 +354,9 @@ def process_translations(content_list): + 'content' logger.warning(msg) index.extend(default_lang_items) - translations.extend(filter( - lambda x: x not in default_lang_items, - items - )) + translations.extend([x for x in items if x not in default_lang_items]) for a in items: - a.translations = filter(lambda x: x != a, items) + a.translations = [x for x in items if x != a] return index, translations @@ -333,6 +415,6 @@ def set_date_tzinfo(d, tz_name=None): def mkdir_p(path): try: os.makedirs(path) - except OSError, e: + except OSError as e: if e.errno != errno.EEXIST: raise diff --git a/pelican/writers.py b/pelican/writers.py index 42ddfb13e8..8374ea3e96 100644 --- a/pelican/writers.py +++ b/pelican/writers.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- -from __future__ import with_statement +from __future__ import with_statement, unicode_literals, print_function +import six import os import locale @@ -57,7 +58,7 @@ def write_feed(self, elements, context, filename=None, feed_type='atom'): :param feed_type: the feed type to use (atom or rss) """ old_locale = locale.setlocale(locale.LC_ALL) - locale.setlocale(locale.LC_ALL, 'C') + locale.setlocale(locale.LC_ALL, str('C')) try: self.site_url = context.get('SITEURL', get_relative_path(filename)) self.feed_domain = context.get('FEED_DOMAIN') @@ -68,7 +69,7 @@ def write_feed(self, elements, context, filename=None, feed_type='atom'): max_items = len(elements) if self.settings['FEED_MAX_ITEMS']: max_items = min(self.settings['FEED_MAX_ITEMS'], max_items) - for i in xrange(max_items): + for i in range(max_items): self._add_item_to_the_feed(feed, elements[i]) if filename: @@ -77,7 +78,7 @@ def write_feed(self, elements, context, filename=None, feed_type='atom'): os.makedirs(os.path.dirname(complete_path)) except Exception: pass - fp = open(complete_path, 'w') + fp = open(complete_path, 'w', encoding='utf-8' if six.PY3 else None) feed.write(fp, 'utf-8') logger.info('writing %s' % complete_path) @@ -108,7 +109,7 @@ def write_file(self, name, template, context, relative_urls=True, def _write_file(template, localcontext, output_path, name): """Render the template write the file.""" old_locale = locale.setlocale(locale.LC_ALL) - locale.setlocale(locale.LC_ALL, 'C') + locale.setlocale(locale.LC_ALL, str('C')) try: output = template.render(localcontext) finally: @@ -120,7 +121,7 @@ def _write_file(template, localcontext, output_path, name): pass with open(filename, 'w', encoding='utf-8') as f: f.write(output) - logger.info(u'writing %s' % filename) + logger.info('writing %s' % filename) localcontext = context.copy() if relative_urls: @@ -135,7 +136,7 @@ def _write_file(template, localcontext, output_path, name): if paginated: # pagination needed, init paginators paginators = {} - for key in paginated.iterkeys(): + for key in paginated.keys(): object_list = paginated[key] if self.settings.get('DEFAULT_PAGINATION'): @@ -147,9 +148,9 @@ def _write_file(template, localcontext, output_path, name): # generated pages, and write name_root, ext = os.path.splitext(name) - for page_num in range(paginators.values()[0].num_pages): + for page_num in range(list(paginators.values())[0].num_pages): paginated_localcontext = localcontext.copy() - for key in paginators.iterkeys(): + for key in paginators.keys(): paginator = paginators[key] page = paginator.page(page_num + 1) paginated_localcontext.update( diff --git a/samples/pelican.conf.py b/samples/pelican.conf.py index 0ac4cd2c1e..714a418cb3 100755 --- a/samples/pelican.conf.py +++ b/samples/pelican.conf.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- -AUTHOR = u'Alexis Métaireau' -SITENAME = u"Alexis' log" +from __future__ import unicode_literals + +AUTHOR = 'Alexis Métaireau' +SITENAME = "Alexis' log" SITEURL = 'http://blog.notmyidea.org' TIMEZONE = "Europe/Paris" @@ -10,7 +12,7 @@ REVERSE_CATEGORY_ORDER = True LOCALE = "C" DEFAULT_PAGINATION = 4 -DEFAULT_DATE = (2012, 03, 02, 14, 01, 01) +DEFAULT_DATE = (2012, 3, 2, 14, 1, 1) FEED_ALL_RSS = 'feeds/all.rss.xml' CATEGORY_FEED_RSS = 'feeds/%s.rss.xml' @@ -19,7 +21,7 @@ ('Filyb', "http://filyb.info/"), ('Libert-fr', "http://www.libert-fr.com"), ('N1k0', "http://prendreuncafe.com/blog/"), - (u'Tarek Ziadé', "http://ziade.org/blog"), + ('Tarek Ziadé', "http://ziade.org/blog"), ('Zubin Mithra', "http://zubin71.wordpress.com/"),) SOCIAL = (('twitter', 'http://twitter.com/ametaireau'), diff --git a/setup.py b/setup.py index b6a544144c..13afd8cf05 100755 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ #!/usr/bin/env python from setuptools import setup -requires = ['feedgenerator', 'jinja2 >= 2.6', 'pygments', 'docutils', 'pytz', - 'blinker', 'unidecode'] +requires = ['feedgenerator>=1.5', 'jinja2 >= 2.6', 'pygments', 'docutils', 'pytz', + 'blinker', 'unidecode', 'six'] try: import argparse # NOQA diff --git a/tests/default_conf.py b/tests/default_conf.py index acb7d9da64..09548fddae 100644 --- a/tests/default_conf.py +++ b/tests/default_conf.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -AUTHOR = u'Alexis Métaireau' -SITENAME = u"Alexis' log" +from __future__ import unicode_literals, print_function +AUTHOR = 'Alexis Métaireau' +SITENAME = "Alexis' log" SITEURL = 'http://blog.notmyidea.org' TIMEZONE = 'UTC' @@ -18,7 +19,7 @@ ('Filyb', "http://filyb.info/"), ('Libert-fr', "http://www.libert-fr.com"), ('N1k0', "http://prendreuncafe.com/blog/"), - (u'Tarek Ziadé', "http://ziade.org/blog"), + ('Tarek Ziadé', "http://ziade.org/blog"), ('Zubin Mithra', "http://zubin71.wordpress.com/"),) SOCIAL = (('twitter', 'http://twitter.com/ametaireau'), diff --git a/tests/support.py b/tests/support.py index 1bbaf47ea6..6011e6cd10 100644 --- a/tests/support.py +++ b/tests/support.py @@ -1,13 +1,15 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function __all__ = [ 'get_article', 'unittest', ] -import cStringIO import os import re import subprocess import sys +from six import StringIO import logging from logging.handlers import BufferingHandler @@ -101,7 +103,7 @@ def decorator(func): def wrapper(*args, **kwargs): saved_stdout = sys.stdout - sys.stdout = cStringIO.StringIO() + sys.stdout = StringIO() try: out = func(*args, **kwargs) diff --git a/tests/test_contents.py b/tests/test_contents.py index a8b9877f8b..eb7b65144a 100644 --- a/tests/test_contents.py +++ b/tests/test_contents.py @@ -113,8 +113,8 @@ def test_datetime(self): page = Page(**page_kwargs) self.assertEqual(page.locale_date, - unicode(dt.strftime(_DEFAULT_CONFIG['DEFAULT_DATE_FORMAT']), - 'utf-8')) + dt.strftime(_DEFAULT_CONFIG['DEFAULT_DATE_FORMAT'])) + page_kwargs['settings'] = dict([(x, _DEFAULT_CONFIG[x]) for x in _DEFAULT_CONFIG]) @@ -131,7 +131,7 @@ def test_datetime(self): import locale as locale_module try: page = Page(**page_kwargs) - self.assertEqual(page.locale_date, u'2015-09-13(\u65e5)') + self.assertEqual(page.locale_date, '2015-09-13(\u65e5)') except locale_module.Error: # The constructor of ``Page`` will try to set the locale to # ``ja_JP.utf8``. But this attempt will failed when there is no diff --git a/tests/test_generators.py b/tests/test_generators.py index fb1dbe9cc1..50f5fe3eb3 100644 --- a/tests/test_generators.py +++ b/tests/test_generators.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function from mock import MagicMock import os @@ -31,7 +32,7 @@ def get_populated_generator(self): settings = get_settings() settings['ARTICLE_DIR'] = 'content' settings['DEFAULT_CATEGORY'] = 'Default' - settings['DEFAULT_DATE'] = (1970, 01, 01) + settings['DEFAULT_DATE'] = (1970, 1, 1) self.generator = ArticlesGenerator(settings.copy(), settings, CUR_DIR, settings['THEME'], None, settings['MARKUP']) @@ -70,18 +71,18 @@ def test_generate_context(self): generator = self.get_populated_generator() articles = self.distill_articles(generator.articles) articles_expected = [ - [u'Article title', 'published', 'Default', 'article'], - [u'Article with markdown and summary metadata single', 'published', u'Default', 'article'], - [u'Article with markdown and summary metadata multi', 'published', u'Default', 'article'], - [u'Article with template', 'published', 'Default', 'custom'], - [u'Test md File', 'published', 'test', 'article'], - [u'Rst with filename metadata', 'published', u'yeah', 'article'], - [u'Test Markdown extensions', 'published', u'Default', 'article'], - [u'This is a super article !', 'published', 'Yeah', 'article'], - [u'This is an article with category !', 'published', 'yeah', 'article'], - [u'This is an article without category !', 'published', 'Default', 'article'], - [u'This is an article without category !', 'published', 'TestCategory', 'article'], - [u'This is a super article !', 'published', 'yeah', 'article'] + ['Article title', 'published', 'Default', 'article'], + ['Article with markdown and summary metadata single', 'published', 'Default', 'article'], + ['Article with markdown and summary metadata multi', 'published', 'Default', 'article'], + ['Article with template', 'published', 'Default', 'custom'], + ['Test md File', 'published', 'test', 'article'], + ['Rst with filename metadata', 'published', 'yeah', 'article'], + ['Test Markdown extensions', 'published', 'Default', 'article'], + ['This is a super article !', 'published', 'Yeah', 'article'], + ['This is an article with category !', 'published', 'yeah', 'article'], + ['This is an article without category !', 'published', 'Default', 'article'], + ['This is an article without category !', 'published', 'TestCategory', 'article'], + ['This is a super article !', 'published', 'yeah', 'article'] ] self.assertItemsEqual(articles_expected, articles) @@ -97,7 +98,7 @@ def test_do_not_use_folder_as_category(self): settings = _DEFAULT_CONFIG.copy() settings['ARTICLE_DIR'] = 'content' settings['DEFAULT_CATEGORY'] = 'Default' - settings['DEFAULT_DATE'] = (1970, 01, 01) + settings['DEFAULT_DATE'] = (1970, 1, 1) settings['USE_FOLDER_AS_CATEGORY'] = False settings['filenames'] = {} generator = ArticlesGenerator(settings.copy(), settings, @@ -179,7 +180,7 @@ def distill_pages(self, pages): def test_generate_context(self): settings = get_settings() settings['PAGE_DIR'] = 'TestPages' - settings['DEFAULT_DATE'] = (1970, 01, 01) + settings['DEFAULT_DATE'] = (1970, 1, 1) generator = PagesGenerator(settings.copy(), settings, CUR_DIR, settings['THEME'], None, @@ -189,14 +190,14 @@ def test_generate_context(self): hidden_pages = self.distill_pages(generator.hidden_pages) pages_expected = [ - [u'This is a test page', 'published', 'page'], - [u'This is a markdown test page', 'published', 'page'], - [u'This is a test page with a preset template', 'published', 'custom'] + ['This is a test page', 'published', 'page'], + ['This is a markdown test page', 'published', 'page'], + ['This is a test page with a preset template', 'published', 'custom'] ] hidden_pages_expected = [ - [u'This is a test hidden page', 'hidden', 'page'], - [u'This is a markdown test hidden page', 'hidden', 'page'], - [u'This is a test hidden page with a custom template', 'hidden', 'custom'] + ['This is a test hidden page', 'hidden', 'page'], + ['This is a markdown test hidden page', 'hidden', 'page'], + ['This is a test hidden page with a custom template', 'hidden', 'custom'] ] self.assertItemsEqual(pages_expected,pages) diff --git a/tests/test_importer.py b/tests/test_importer.py index 449a7643b2..a79cd49384 100644 --- a/tests/test_importer.py +++ b/tests/test_importer.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function import os @@ -9,7 +10,7 @@ WORDPRESS_XML_SAMPLE = os.path.join(CUR_DIR, 'content', 'wordpressexport.xml') try: - import BeautifulSoup + from bs4 import BeautifulSoup except ImportError: BeautifulSoup = False # NOQA @@ -48,26 +49,6 @@ def test_can_toggle_raw_html_code_parsing(self): strip_raw=True)) self.assertFalse(any('Multi-line metadata should be'\ - u' supported\nas well as inline'\ - u' markup.

\n', + 'summary': '

Multi-line metadata should be'\ + ' supported\nas well as inline'\ + ' markup.

\n', 'date': datetime.datetime(2010, 12, 2, 10, 14), 'tags': ['foo', 'bar', 'foobar'], 'custom_field': 'http://notmyidea.org', @@ -40,7 +41,7 @@ def test_article_with_filename_metadata(self): settings={}) expected = { 'category': 'yeah', - 'author': u'Alexis Métaireau', + 'author': 'Alexis Métaireau', 'title': 'Rst with filename metadata', } for key, value in metadata.items(): @@ -53,7 +54,7 @@ def test_article_with_filename_metadata(self): }) expected = { 'category': 'yeah', - 'author': u'Alexis Métaireau', + 'author': 'Alexis Métaireau', 'title': 'Rst with filename metadata', 'date': datetime.datetime(2012, 11, 29), } @@ -69,7 +70,7 @@ def test_article_with_filename_metadata(self): }) expected = { 'category': 'yeah', - 'author': u'Alexis Métaireau', + 'author': 'Alexis Métaireau', 'title': 'Rst with filename metadata', 'date': datetime.datetime(2012, 11, 29), 'slug': 'article_with_filename_metadata', @@ -101,7 +102,7 @@ def test_typogrify(self): # otherwise, typogrify should be applied content, _ = readers.read_file(_filename('article.rst'), settings={'TYPOGRIFY': True}) - expected = u"

This is some content. With some stuff to "\ + expected = "

This is some content. With some stuff to "\ "“typogrify”.

\n

Now with added "\ 'support for '\ 'TLA.

\n' @@ -168,7 +169,7 @@ def test_article_with_filename_metadata(self): settings={}) expected = { 'category': 'yeah', - 'author': u'Alexis Métaireau', + 'author': 'Alexis Métaireau', } for key, value in expected.items(): self.assertEquals(value, metadata[key], key) @@ -180,7 +181,7 @@ def test_article_with_filename_metadata(self): }) expected = { 'category': 'yeah', - 'author': u'Alexis Métaireau', + 'author': 'Alexis Métaireau', 'date': datetime.datetime(2012, 11, 30), } for key, value in expected.items(): @@ -195,7 +196,7 @@ def test_article_with_filename_metadata(self): }) expected = { 'category': 'yeah', - 'author': u'Alexis Métaireau', + 'author': 'Alexis Métaireau', 'date': datetime.datetime(2012, 11, 30), 'slug': 'md_w_filename_meta', 'mymeta': 'foo', @@ -203,20 +204,6 @@ def test_article_with_filename_metadata(self): for key, value in expected.items(): self.assertEquals(value, metadata[key], key) - @unittest.skipUnless(readers.Markdown, "markdown isn't installed") - def test_article_with_summary_metadata(self): - reader = readers.MarkdownReader({}) - content, metadata = reader.read( - _filename('article_with_markdown_and_summary_metadata_single.md')) - expected_summary = u'

A single-line summary should be supported'\ - u' as well as inline markup.

' - self.assertEquals(expected_summary, metadata['summary'], 'summary') - content, metadata = reader.read( - _filename('article_with_markdown_and_summary_metadata_multi.md')) - expected_summary = u'

A multi-line summary should be supported'\ - u'\nas well as inline markup.

' - self.assertEquals(expected_summary, metadata['summary'], 'summary') - class AdReaderTest(unittest.TestCase): @unittest.skipUnless(readers.asciidoc, "asciidoc isn't installed") diff --git a/tests/test_settings.py b/tests/test_settings.py index 379e3deb30..0e610c55cc 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function import copy from os.path import dirname, abspath, join @@ -16,7 +18,7 @@ def setUp(self): self.settings = read_settings(default_conf) def test_overwrite_existing_settings(self): - self.assertEqual(self.settings.get('SITENAME'), u"Alexis' log") + self.assertEqual(self.settings.get('SITENAME'), "Alexis' log") self.assertEqual(self.settings.get('SITEURL'), 'http://blog.notmyidea.org') diff --git a/tests/test_utils.py b/tests/test_utils.py index eda1388a08..eddb3e25de 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function import shutil import os import datetime @@ -41,10 +42,10 @@ def test_slugify(self): samples = (('this is a test', 'this-is-a-test'), ('this is a test', 'this-is-a-test'), - (u'this → is ← a ↑ test', 'this-is-a-test'), + ('this → is ← a ↑ test', 'this-is-a-test'), ('this--is---a test', 'this-is-a-test'), - (u'unicode測試許功蓋，你看到了嗎？', 'unicodece-shi-xu-gong-gai-ni-kan-dao-liao-ma'), - (u'大飯原発４号機、１８日夜起動へ', 'da-fan-yuan-fa-4hao-ji-18ri-ye-qi-dong-he'),) + ('unicode測試許功蓋，你看到了嗎？', 'unicodece-shi-xu-gong-gai-ni-kan-dao-liao-ma'), + ('大飯原発４号機、１８日夜起動へ', 'da-fan-yuan-fa-4hao-ji-18ri-ye-qi-dong-he'),) for value, expected in samples: self.assertEquals(utils.slugify(value), expected) diff --git a/tests/test_webassets.py b/tests/test_webassets.py index ceba77758f..87b2789c37 100644 --- a/tests/test_webassets.py +++ b/tests/test_webassets.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# from __future__ import unicode_literals import hashlib import os diff --git a/tox.ini b/tox.ini index f7d9cf8211..8c3961b97f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,15 +1,74 @@ +# This tests the unified codebase (py26-py32) of Pelican. +# depends on some external libraries that aren't released yet. +# +# To run Pelican, you will already have checked out and installed them. +# +# Now we must tell tox about this package, otherwise tox would load the old +# libraries from PyPi. +# +# Run tox from the libraries source tree. It will save its package in +# the distshare directory from where the tests here will pick it up. +# +# Do that for +# https://github.com/dmdm/smartypants.git +# +# and typogrify: +# https://github.com/dmdm/typogrify/tree/py3k +# +# and webassets: +# https://github.com/dmdm/webassets/tree/py3k +# +# +# CAVEAT: +# ------- +# +# 1/ +# Please be aware that my ports of typogrify and webassets are just 2to3'd. +# They are not backwards compatible with Python 2. +# +# 2/ +# Webassets still has unresolved issues, so I deactivated it for Py32 tests. + + [tox] -envlist = py26,py27 +envlist = py26,py27,py32 [testenv] commands = - nosetests -s tests unit2 discover [] + nosetests -s tests +deps = + +[testenv:py26] deps = nose unittest2 mock Markdown - BeautifulSoup + BeautifulSoup4 + feedgenerator typogrify webassets + +[testenv:py27] +deps = + nose + unittest2 + mock + Markdown + BeautifulSoup4 + feedgenerator + typogrify + webassets + +[testenv:py32] +deps = + nose + unittest2py3k + mock + Markdown + BeautifulSoup4 + feedgenerator +# {distshare}/smartypants-1.6.0.3.zip +# {distshare}/typogrify-2.0.0.zip +# {distshare}/webassets-0.8.dev.zip