merged master

CHANGES.txt

@@ -1,11 +1,24 @@
 New in master
-============
+=============
 
 Features
 --------
 
-* New Oembed-based media shortcode (Issue #2170)
 * New Pygal-based chart shortcode (Issue #2170)
+* Add ``post_type`` to post-list directive (Issue #2272)
+* Use ``sys.executable`` for launching pip in ``plugin`` (Issue #2275)
+
+Bugfixes
+--------
+
+* Fix a JSON conversion bug in the WordPress importer (Issue #2264)
+* Don’t create download directories when not downloading WordPress
+  attachments (Issue #2260)
+* Don’t display “Good link” messages in ``nikola check -l`` by default,
+  can be re-enabled with ``-v`` option (Issue #2268)
+* Fix a format string in ``nikola check`` (Issue #2267)
+* Don't crash wordpress importer when posts are "empty" (Issue #2263)
+* Don't put untranslated and nonexistant posts in sitemap (Issue #2289)
 
 New in v7.7.6
 =============

docs/extending.txt

@@ -9,7 +9,7 @@
 Extending Nikola
 ================
 
-:Version: v7.7.6
+:Version: 7.7.6
 :Author: Roberto Alsina <ralsina@netmanagers.com.ar>
 
 .. class:: alert alert-info pull-right

docs/man/nikola.rst

@@ -6,7 +6,7 @@ Nikola
 A Static Site and Blog Generator
 --------------------------------
 
-:Version: Nikola vv7.7.6
+:Version: Nikola v7.7.6
 :Manual section: 1
 :Manual group: User Commands
 

docs/manual.txt

@@ -9,7 +9,7 @@
 The Nikola Handbook
 ===================
 
-:Version: v7.7.6
+:Version: 7.7.6
 
 .. class:: alert alert-info pull-right
 

docs/social_buttons.txt

@@ -9,7 +9,7 @@
 Using Alternative Social Buttons with Nikola
 ============================================
 
-:Version: v7.7.6
+:Version: 7.7.6
 
 .. class:: alert alert-info pull-right
 

docs/sphinx/conf.py

@@ -54,9 +54,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = 'v7.7.6'
+version = '7.7.6'
 # The full version, including alpha/beta/rc tags.
-release = 'v7.7.6'
+release = '7.7.6'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/theming.txt

@@ -9,7 +9,7 @@
 Theming Nikola
 ==============
 
-:Version: v7.7.6
+:Version: 7.7.6
 :Author: Roberto Alsina <ralsina@netmanagers.com.ar>
 
 .. class:: alert alert-info pull-right

nikola/__init__.py

@@ -29,7 +29,7 @@
 from __future__ import absolute_import
 import os
 
-__version__ = "v7.7.6"
+__version__ = "7.7.6"
 DEBUG = bool(os.getenv('NIKOLA_DEBUG'))
 
 from .nikola import Nikola  # NOQA

nikola/plugins/basic_import.py

@@ -127,9 +127,12 @@ def transform_content(cls, content):
     def write_content(cls, filename, content, rewrite_html=True):
         """Write content to file."""
         if rewrite_html:
-            doc = html.document_fromstring(content)
-            doc.rewrite_links(replacer)
-            content = html.tostring(doc, encoding='utf8')
+            try:
+                doc = html.document_fromstring(content)
+                doc.rewrite_links(replacer)
+                content = html.tostring(doc, encoding='utf8')
+            except etree.ParserError:
+                content = content.encode('utf-8')
         else:
             content = content.encode('utf-8')
 

nikola/plugins/command/check.py

@@ -32,6 +32,7 @@
 import re
 import sys
 import time
+import logbook
 try:
     from urllib import unquote
     from urlparse import urlparse, urljoin, urldefrag
@@ -164,9 +165,9 @@ def _execute(self, options, args):
             print(self.help())
             return False
         if options['verbose']:
-            self.logger.level = 1
+            self.logger.level = logbook.DEBUG
         else:
-            self.logger.level = 4
+            self.logger.level = logbook.NOTICE
         failure = False
         if options['links']:
             failure |= self.scan_links(options['find_sources'], options['remote'])
@@ -343,7 +344,7 @@ def analyze(self, fname, find_sources=False, check_remote=False):
 
                 elif target_filename not in self.existing_targets:
                     if os.path.exists(target_filename):
-                        self.logger.notice(u"Good link {0} => {1}".format(target, target_filename))
+                        self.logger.info("Good link {0} => {1}".format(target, target_filename))
                         self.existing_targets.add(target_filename)
                     else:
                         rv = True

nikola/plugins/command/import_wordpress.py

@@ -382,7 +382,7 @@ def read_xml_file(cls, filename):
                 if b'<atom:link rel=' in line:
                     continue
                 xml.append(line)
-        return b'\n'.join(xml)
+        return b''.join(xml)
 
     @classmethod
     def get_channel_from_file(cls, filename):
@@ -446,9 +446,6 @@ def populate_context(self, channel):
 
     def download_url_content_to_file(self, url, dst_path):
         """Download some content (attachments) to a file."""
-        if self.no_downloads:
-            return
-
         try:
             request = requests.get(url, auth=self.auth)
             if request.status_code >= 400:
@@ -468,10 +465,13 @@ def import_attachment(self, item, wordpress_namespace):
                             'foo')
         path = urlparse(url).path
         dst_path = os.path.join(*([self.output_folder, 'files'] + list(path.split('/'))))
-        dst_dir = os.path.dirname(dst_path)
-        utils.makedirs(dst_dir)
-        LOGGER.info("Downloading {0} => {1}".format(url, dst_path))
-        self.download_url_content_to_file(url, dst_path)
+        if self.no_downloads:
+            LOGGER.info("Skipping downloading {0} => {1}".format(url, dst_path))
+        else:
+            dst_dir = os.path.dirname(dst_path)
+            utils.makedirs(dst_dir)
+            LOGGER.info("Downloading {0} => {1}".format(url, dst_path))
+            self.download_url_content_to_file(url, dst_path)
         dst_url = '/'.join(dst_path.split(os.sep)[2:])
         links[link] = '/' + dst_url
         links[url] = '/' + dst_url
@@ -562,15 +562,18 @@ def add(our_key, wp_key, is_int=False, ignore_zero=False, is_float=False):
                         meta = {}
                         meta['size'] = size.decode('utf-8')
                         if width_key in metadata[size_key][size] and height_key in metadata[size_key][size]:
-                            meta['width'] = metadata[size_key][size][width_key]
-                            meta['height'] = metadata[size_key][size][height_key]
+                            meta['width'] = int(metadata[size_key][size][width_key])
+                            meta['height'] = int(metadata[size_key][size][height_key])
 
                         path = urlparse(url).path
                         dst_path = os.path.join(*([self.output_folder, 'files'] + list(path.split('/'))))
-                        dst_dir = os.path.dirname(dst_path)
-                        utils.makedirs(dst_dir)
-                        LOGGER.info("Downloading {0} => {1}".format(url, dst_path))
-                        self.download_url_content_to_file(url, dst_path)
+                        if self.no_downloads:
+                            LOGGER.info("Skipping downloading {0} => {1}".format(url, dst_path))
+                        else:
+                            dst_dir = os.path.dirname(dst_path)
+                            utils.makedirs(dst_dir)
+                            LOGGER.info("Downloading {0} => {1}".format(url, dst_path))
+                            self.download_url_content_to_file(url, dst_path)
                         dst_url = '/'.join(dst_path.split(os.sep)[2:])
                         links[url] = '/' + dst_url
 

nikola/plugins/command/plugin.py

@@ -29,6 +29,7 @@
 from __future__ import print_function
 import io
 import os
+import sys
 import shutil
 import subprocess
 import time
@@ -254,7 +255,7 @@ def do_install(self, url, name, show_install_notes=True):
             LOGGER.notice('This plugin has Python dependencies.')
             LOGGER.info('Installing dependencies with pip...')
             try:
-                subprocess.check_call(('pip', 'install', '-r', reqpath))
+                subprocess.check_call((sys.executable, '-m', 'pip', 'install', '-r', reqpath))
             except subprocess.CalledProcessError:
                 LOGGER.error('Could not install the dependencies.')
                 print('Contents of the requirements.txt file:\n')

nikola/plugins/task/sitemap/__init__.py

@@ -158,7 +158,7 @@ def scan_locs():
                         continue
                     alternates = []
                     if post:
-                        for lang in kw['translations']:
+                        for lang in post.translated_to:
                             alt_url = post.permalink(lang=lang, absolute=True)
                             if encodelink(loc) == alt_url:
                                 continue
@@ -215,7 +215,7 @@ def scan_locs():
                         loc = urljoin(base_url, base_path + path)
                         alternates = []
                         if post:
-                            for lang in kw['translations']:
+                            for lang in post.translated_to:
                                 alt_url = post.permalink(lang=lang, absolute=True)
                                 if encodelink(loc) == alt_url:
                                     continue

requirements-tests.txt

@@ -1,8 +1,8 @@
 -r requirements-extras.txt
 mock==1.3.0
 coverage==4.0.3
-pytest==2.8.7
+pytest==2.9.0
 pytest-cov==2.2.1
 freezegun==0.3.6
-python-coveralls==2.6.0
+python-coveralls==2.7.0
 colorama>=0.3.4

scripts/github-release.py

@@ -1,19 +1,27 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import subprocess
-import sys
 import os
+import argparse
 
 if not os.path.exists('.pypt/gh-token'):
     print("To use this script, you must create a GitHub token first.")
     print("Get a token here: https://github.com/settings/tokens")
     print("Then, put it in a file named .pypt/gh-token")
     exit(1)
 
-inpf = input if sys.version_info[0] == 3 else raw_input
+parser = argparse.ArgumentParser(description="GitHub Release helper")
+parser.add_argument("FILE", nargs=1, help="Markdown file to use")
+parser.add_argument("TAG", nargs=1, help="Tag name (usually vX.Y.Z)")
+
+args = parser.parse_args()
+
+if not args.TAG[0].startswith("v"):
+    print("WARNING: tag should start with v")
+    i = input("Add `v` to tag? [y/n] ")
+    if i.lower().strip().startswith('y'):
+        args.TAG[0] = 'v' + args.TAG[0]
 
-FILE = inpf("Markdown file to use: ")
 BASEDIR = os.getcwd()
 REPO = 'getnikola/nikola'
-TAG = inpf("Tag name (usually vX.Y.Z): ")
 
-subprocess.call(['.pypt/ghrel', FILE, BASEDIR, REPO, TAG])
+subprocess.call(['.pypt/ghrel', args.FILE[0], BASEDIR, REPO, args.TAG[0]])

scripts/set_version.py

@@ -26,7 +26,12 @@ def sed_like_thing(pattern, repl, path):
 
 if __name__ == "__main__":
     inpf = raw_input if sys.version_info[0] == 2 else input
-    version = inpf("New version number (in format X.Y.Z): ").strip()
+    while True:
+        version = inpf("New version number (in format X.Y.Z): ").strip()
+        if version.startswith('v'):
+            print("ERROR: the version number must not start with v.")
+        else:
+            break
 
     for doc in glob.glob(os.path.join("docs/*.txt")):
         sed_like_thing(":Version: .*", ":Version: {0}".format(version), doc)

setup.py

@@ -113,7 +113,7 @@ def run(self):
 
 
 setup(name='Nikola',
-      version='v7.7.6',
+      version='7.7.6',
       description='A modular, fast, simple, static website and blog generator',
       long_description=open('README.rst').read(),
       author='Roberto Alsina and others',

tests/test_command_import_wordpress.py

@@ -238,20 +238,18 @@ def test_importing_posts_and_attachments(self):
 ```Python
 
 import sys
-
 print sys.version
 
 ```
 
 The end.
-
 """, True)
 
         self.assertTrue(write_attachments_info.called)
         write_attachments_info.assert_any_call('new_site/posts/2008/07/arzt-und-pfusch-s-i-c-k.attachments.json'.replace('/', os.sep),
                                                {10: {'wordpress_user_name': 'Niko',
                                                      'files_meta': [{'width': 300, 'height': 299},
-                                                                    {'width': b'150', 'size': 'thumbnail', 'height': b'150'}],
+                                                                    {'width': 150, 'size': 'thumbnail', 'height': 150}],
                                                      'excerpt': 'Arzt+Pfusch - S.I.C.K.',
                                                      'date_utc': '2009-07-16 19:40:37',
                                                      'content': 'Das Cover von Arzt+Pfusch - S.I.C.K.',
@@ -262,27 +260,17 @@ def test_importing_posts_and_attachments(self):
         write_content.assert_any_call(
             'new_site/posts/2008/07/arzt-und-pfusch-s-i-c-k.md'.replace('/', os.sep),
             '''<img class="size-full wp-image-10 alignright" title="Arzt+Pfusch - S.I.C.K." src="http://some.blog/wp-content/uploads/2008/07/arzt_und_pfusch-sick-cover.png" alt="Arzt+Pfusch - S.I.C.K." width="210" height="209" />Arzt+Pfusch - S.I.C.K.Gerade bin ich \xfcber das Album <em>S.I.C.K</em> von <a title="Arzt+Pfusch" href="http://www.arztpfusch.com/" target="_blank">Arzt+Pfusch</a> gestolpert, welches Arzt+Pfusch zum Download f\xfcr lau anbieten. Das Album steht unter einer Creative Commons <a href="http://creativecommons.org/licenses/by-nc-nd/3.0/de/">BY-NC-ND</a>-Lizenz.
-
 Die Ladung <em>noisebmstupidevildustrial</em> gibts als MP3s mit <a href="http://www.archive.org/download/dmp005/dmp005_64kb_mp3.zip">64kbps</a> und <a href="http://www.archive.org/download/dmp005/dmp005_vbr_mp3.zip">VBR</a>, als Ogg Vorbis und als FLAC (letztere <a href="http://www.archive.org/details/dmp005">hier</a>). <a href="http://www.archive.org/download/dmp005/dmp005-artwork.zip">Artwork</a> und <a href="http://www.archive.org/download/dmp005/dmp005-lyrics.txt">Lyrics</a> gibts nochmal einzeln zum Download.''', True)
         write_content.assert_any_call(
             'new_site/stories/kontakt.md'.replace('/', os.sep), """<h1>Datenschutz</h1>
-
 Ich erhebe und speichere automatisch in meine Server Log Files Informationen, die dein Browser an mich \xfcbermittelt. Dies sind:
-
 <ul>
-
     <li>Browsertyp und -version</li>
-
     <li>verwendetes Betriebssystem</li>
-
     <li>Referrer URL (die zuvor besuchte Seite)</li>
-
     <li>IP Adresse des zugreifenden Rechners</li>
-
     <li>Uhrzeit der Serveranfrage.</li>
-
 </ul>
-
 Diese Daten sind f\xfcr mich nicht bestimmten Personen zuordenbar. Eine Zusammenf\xfchrung dieser Daten mit anderen Datenquellen wird nicht vorgenommen, die Daten werden einzig zu statistischen Zwecken erhoben.""", True)
 
         self.assertTrue(len(self.import_command.url_map) > 0)