merged master

CHANGES.txt

@@ -4,14 +4,18 @@ New in master
 Features
 --------
 
+* Cleaner formatting of HTML output
 * Allowing category hierarchies via new option CATEGORY_ALLOW_HIERARCHIES
   (Issue #1520)
 
 Bugfixes
 --------
 
 * Use a global directory for gallery images, ignoring translations (Issue #1726)
-* Exclude ``<meta content="noindex" name="robots">`` from sitemaps
+* The post-list directive only triggers rebuilds if timeline changes (Issue #1688)
+* Don’t check the same remote URL more than once (Issue #1732)
+* All remotely checked links resulted in incorrect warnings (`nikola check -lr`)
+* Exclude `<meta content="noindex" name="robots">` from sitemaps
 * new_post paths are now relative to CWD (Issue #1325)
 
 New in v7.4.1

docs/extending.txt

@@ -407,9 +407,11 @@ Currently Nikola emits the following signals:
     When all the configuration file is processed. Note that plugins are activated before this is emitted.
 ``scanned``
     After posts are scanned.
-``new_post``
-    When a new post is created, using the ``nikola new_post`` command.  The signal
+``new_post`` / ``new_page``
+    When a new post is created, using the ``nikola new_post``/``nikola new_page`` commands.  The signal
     data contains the path of the file, and the metadata file (if there is one).
+``existing_post`` / ``existing_page``
+    When a new post fails to be created due to a title conflict. Contains the same data as ``new_post``.
 ``deployed``
     When the ``nikola deploy`` command is run, and there is at least one new
     entry/post since ``last_deploy``.  The signal data is of the form::

docs/manual.txt

@@ -1655,9 +1655,12 @@ Post List
 
 .. WARNING::
 
-   Any post or page that uses this directive will **never** be considered up-to-date,
-   meaning that every time you build the site, that post/page and everything that
-   depends on it (tag pages, RSS feeds, sitemap, etc.) will be rebuilt.
+   Any post or page that uses this directive will be considered out of date,
+   every time a post is added or deleted, causing maybe unnecessary rebuilds.
+
+   On the other hand, it will sometimes **not** be considered out of date if
+   a post content changes, so it can sometimes be shown outdated, in those
+   cases, use ``nikola build -a`` to force a total rebuild.
 
 
 This directive can be used to generate a list of posts. You could use it, for

nikola/data/themes/bootstrap-jinja/templates/base_helper.tmpl

@@ -69,15 +69,15 @@ lang="{{ lang }}">
 {% macro late_load_js() %}
     {% if use_bundles %}
         {% if use_cdn %}
-            <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+            <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script>
             <script src="//maxcdn.bootstrapcdn.com/twitter-bootstrap/2.3.2/js/bootstrap.min.js"></script>
             <script src="/assets/js/all.js"></script>
         {% else %}
             <script src="/assets/js/all-nocdn.js"></script>
         {% endif %}
     {% else %}
         {% if use_cdn %}
-            <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+            <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script>
             <script src="//maxcdn.bootstrapcdn.com/twitter-bootstrap/2.3.2/js/bootstrap.min.js"></script>
         {% else %}
             <script src="/assets/js/jquery.min.js"></script>

nikola/data/themes/bootstrap/templates/base_helper.tmpl

@@ -69,15 +69,15 @@ lang="${lang}">
 <%def name="late_load_js()">
     %if use_bundles:
         %if use_cdn:
-            <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+            <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script>
             <script src="//maxcdn.bootstrapcdn.com/twitter-bootstrap/2.3.2/js/bootstrap.min.js"></script>
             <script src="/assets/js/all.js"></script>
         %else:
             <script src="/assets/js/all-nocdn.js"></script>
         %endif
     %else:
         %if use_cdn:
-            <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+            <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script>
             <script src="//maxcdn.bootstrapcdn.com/twitter-bootstrap/2.3.2/js/bootstrap.min.js"></script>
         %else:
             <script src="/assets/js/jquery.min.js"></script>

nikola/data/themes/bootstrap3-jinja/templates/base_helper.tmpl

@@ -68,15 +68,15 @@ lang="{{ lang }}">
 {% macro late_load_js() %}
     {% if use_bundles %}
         {% if use_cdn %}
-            <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
+            <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script>
             <script src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
             <script src="/assets/js/all.js"></script>
         {% else %}
             <script src="/assets/js/all-nocdn.js"></script>
         {% endif %}
     {% else %}
         {% if use_cdn %}
-            <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
+            <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script>
             <script src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
         {% else %}
             <script src="/assets/js/jquery.min.js"></script>

nikola/data/themes/bootstrap3/templates/base_helper.tmpl

@@ -68,15 +68,15 @@ lang="${lang}">
 <%def name="late_load_js()">
     %if use_bundles:
         %if use_cdn:
-            <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
+            <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script>
             <script src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
             <script src="/assets/js/all.js"></script>
         %else:
             <script src="/assets/js/all-nocdn.js"></script>
         %endif
     %else:
         %if use_cdn:
-            <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
+            <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script>
             <script src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
         %else:
             <script src="/assets/js/jquery.min.js"></script>

nikola/nikola.py

@@ -969,7 +969,8 @@ def render_template(self, template_name, output_name, context):
         src = "/".join(src.split(os.sep))
 
         utils.makedirs(os.path.dirname(output_name))
-        doc = lxml.html.document_fromstring(data)
+        parser = lxml.html.HTMLParser(remove_blank_text=True)
+        doc = lxml.html.document_fromstring(data, parser)
         doc.rewrite_links(lambda dst: self.url_replacer(src, dst, context['lang']))
         data = b'<!DOCTYPE html>\n' + lxml.html.tostring(doc, encoding='utf8', method='html', pretty_print=True)
         with open(output_name, "wb+") as post_file:

nikola/plugins/command/check.py

@@ -173,6 +173,7 @@ def _execute(self, options, args):
             sys.exit(1)
 
     existing_targets = set([])
+    checked_remote_targets = {}
 
     def analyze(self, task, find_sources=False, check_remote=False):
         rv = False
@@ -215,13 +216,24 @@ def analyze(self, task, find_sources=False, check_remote=False):
                         ((parsed.scheme or target.startswith('//')) and url_type in ('rel_path', 'full_path')):
                     if not check_remote or parsed.scheme not in ["http", "https"]:
                         continue
-                    if parsed.netloc == base_url.netloc:
+                    if parsed.netloc == base_url.netloc:  # absolute URL to self.site
+                        continue
+                    if target in self.checked_remote_targets:  # already checked this exact target
+                        if self.checked_remote_targets[target] > 399:
+                            self.logger.warn("Broken link in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target]))
                         continue
                     # Check the remote link works
-                    resp = requests.head(target)
+                    req_headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0 (Nikola)'}  # I’m a real boy!
+                    resp = requests.head(target, headers=req_headers)
+                    self.checked_remote_targets[target] = resp.status_code
                     if resp.status_code > 399:  # Error
                         self.logger.warn("Broken link in {0}: {1} [Error {2}]".format(filename, target, resp.status_code))
                         continue
+                    elif resp.status_code <= 399:  # The address leads *somewhere* that is not an error
+                        self.logger.debug("Successfully checked remote link in {0}: {1} [HTTP: {2}]".format(filename, target, resp.status_code))
+                        continue
+                    self.logger.warn("Could not check remote link in {0}: {1} [Unknown problem]".format(filename, target))
+                    continue
 
                 if url_type == 'rel_path':
                     target_filename = os.path.abspath(

nikola/plugins/compile/rest/post_list.py

@@ -27,12 +27,11 @@
 
 import os
 import uuid
+import natsort
 
 from docutils import nodes
 from docutils.parsers.rst import Directive, directives
 
-from natsort import natsorted
-
 from nikola import utils
 from nikola.plugin_categories import RestExtension
 
@@ -160,7 +159,7 @@ def run(self):
             filtered_timeline.append(post)
 
         if sort:
-            filtered_timeline = natsorted(filtered_timeline, key=lambda post: post.meta[lang][sort])
+            filtered_timeline = natsort.natsorted(filtered_timeline, key=lambda post: post.meta[lang][sort], alg=natsort.ns.F | natsort.ns.IC)
 
         for post in filtered_timeline[start:stop:step]:
             if slugs:

nikola/plugins/task/posts.py

@@ -40,10 +40,6 @@ def update_deps(post, lang, task):
     task.file_dep.update([p for p in post.fragment_deps(lang) if not p.startswith("####MAGIC####")])
 
 
-def dependence_on_timeline(post, lang):
-    return "####MAGIC####TIMELINE" not in post.fragment_deps(lang)
-
-
 class RenderPosts(Task):
     """Build HTML fragments from metadata and text."""
 
@@ -59,13 +55,25 @@ def gen_tasks(self):
             "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'],
             "demote_headers": self.site.config['DEMOTE_HEADERS'],
         }
+        self.tl_changed = False
 
         yield self.group_task()
 
+        def tl_ch():
+            self.tl_changed = True
+
+        yield {
+            'basename': self.name,
+            'name': 'timeline_changes',
+            'actions': [tl_ch],
+            'uptodate': [utils.config_changed({1: kw['timeline']})],
+        }
+
         for lang in kw["translations"]:
             deps_dict = copy(kw)
             deps_dict.pop('timeline')
             for post in kw['timeline']:
+
                 dest = post.translated_base_path(lang)
                 file_dep = [p for p in post.fragment_deps(lang) if not p.startswith("####MAGIC####")]
                 task = {
@@ -79,7 +87,15 @@ def gen_tasks(self):
                     'clean': True,
                     'uptodate': [
                         utils.config_changed(deps_dict, 'nikola.plugins.task.posts'),
-                        lambda p=post, l=lang: dependence_on_timeline(p, l)
+                        lambda p=post, l=lang: self.dependence_on_timeline(p, l)
                     ] + post.fragment_deps_uptodate(lang),
+                    'task_dep': ['render_posts:timeline_changes']
                 }
                 yield task
+
+    def dependence_on_timeline(self, post, lang):
+        if "####MAGIC####TIMELINE" not in post.fragment_deps(lang):
+            return True  # No dependency on timeline
+        elif self.tl_changed:
+            return False  # Timeline changed
+        return True

nikola/plugins/task/tags.py

@@ -114,8 +114,8 @@ def gen_tasks(self):
             if slug in categories:
                 other_category = categories[slug]
                 utils.LOGGER.error('You have categories that are too similar: {0} and {1}'.format(category, other_category))
-                utils.LOGGER.error('Category {0} is used in: {1}'.format(category, ', '.join([p.source_path for p in self.posts_per_category[category]])))
-                utils.LOGGER.error('Category {0} is used in: {1}'.format(other_category, ', '.join([p.source_path for p in self.posts_per_category[other_category]])))
+                utils.LOGGER.error('Category {0} is used in: {1}'.format(category, ', '.join([p.source_path for p in self.site.posts_per_category[category]])))
+                utils.LOGGER.error('Category {0} is used in: {1}'.format(other_category, ', '.join([p.source_path for p in self.site.posts_per_category[other_category]])))
                 sys.exit(1)
             categories[slug] = category
 
@@ -270,14 +270,14 @@ def page_path(i, displayed_i, num_pages, force_addition, extension=None):
             return utils.adjust_name_for_index_path(self.site.path(kind + feed, tag, lang), i, displayed_i, lang, self.site, force_addition, extension)
 
         context_source = {}
+        title = self._get_title(tag, is_category)
         if kw["generate_rss"]:
             # On a tag page, the feeds include the tag's feeds
             rss_link = ("""<link rel="alternate" type="application/rss+xml" """
                         """type="application/rss+xml" title="RSS for tag """
                         """{0} ({1})" href="{2}">""".format(
-                            tag, lang, self.site.link(kind + "_rss", tag, lang)))
+                            title, lang, self.site.link(kind + "_rss", tag, lang)))
             context_source['rss_link'] = rss_link
-        title = self._get_title(tag, is_category)
         if is_category:
             context_source["category"] = tag
             context_source["category_path"] = self.site.parse_category_name(tag)

nikola/post.py

@@ -29,6 +29,8 @@
 import io
 from collections import defaultdict
 import datetime
+import hashlib
+import json
 import os
 import re
 import string
@@ -229,7 +231,12 @@ def __init__(
         self.compiler.register_extra_dependencies(self)
 
     def __repr__(self):
-        return '<Post: {0}>'.format(self.source_path)
+        # Calculate a hash that represents most data about the post
+        m = hashlib.md5()
+        # source_path modification date (to avoid reading it)
+        m.update(utils.unicode_str(os.stat(self.source_path).st_mtime).encode('utf-8'))
+        m.update(utils.unicode_str(json.dumps(self.meta, cls=utils.CustomEncoder, sort_keys=True)).encode('utf-8'))
+        return '<Post: {0} {1}>'.format(self.source_path, m.hexdigest())
 
     def _has_pretty_url(self, lang):
         if self.pretty_urls and \

nikola/utils.py

@@ -57,7 +57,7 @@
 
 from nikola import DEBUG
 
-__all__ = ['get_theme_path', 'get_theme_chain', 'load_messages', 'copy_tree',
+__all__ = ['CustomEncoder', 'get_theme_path', 'get_theme_chain', 'load_messages', 'copy_tree',
            'copy_file', 'slugify', 'unslugify', 'to_datetime', 'apply_filters',
            'config_changed', 'get_crumbs', 'get_tzname', 'get_asset_path',
            '_reload', 'unicode_str', 'bytes_str', 'unichr', 'Functionary',

requirements.txt

@@ -6,7 +6,7 @@ docutils>=0.12
 mako>=1.0.0
 unidecode>=0.04.16
 lxml>=3.3.5
-yapsy>=1.10.423
+yapsy==1.10.423
 PyRSS2Gen>=1.1
 logbook>=0.7.0
 blinker>=1.3

scripts/jinjify.py

@@ -78,10 +78,10 @@ def jinjify(in_theme, out_theme):
         parent = mappings[child]
 
     with io.open(os.path.join(out_theme, "parent"), "w+", encoding='utf-8') as outf:
-        outf.write(parent + '\n')
+        outf.write(unicode(parent) + u'\n')
 
     with io.open(os.path.join(out_theme, "engine"), "w+", encoding='utf-8') as outf:
-        outf.write("jinja\n")
+        outf.write(u"jinja\n")
 
     # Copy assets
     # shutil.rmtree(os.path.join(out_theme, "assets"))