Merge branch 'master' into task-atom

CHANGES.txt

@@ -11,6 +11,11 @@ Features
 * Atom feed auto-discovery in HTML indexes and category/tag indexes
 * .atom included in the sitemap index
 * New post metadata "updated", inherits "date" if unset
+* Set default new site URL to https://example.com/
+* Plugins can manipulate task dependencies (Issue #1679)
+* LINK_CHECK_WHITELIST now works with output relative and full fs paths
+* Four new filters: html_tidy_nowrap, html_tidy_wrap, html_tidy_wrap_attr,
+  and html_tidy_mini for prettification and minification. Requires tidy5.
 * Multilingual sitemaps (Issue #1610)
 * Compatibility with doit v0.28.0 (Issue #1655)
 * AddThis is no longer added by default to users’ sites
@@ -27,6 +32,7 @@ Bugfixes
 --------
 
 * RSS_LINKS_APPEND_QUERY not working in RSS feeds for tags
+* `nikola check -l` didn’t scan posts
 * Don’t use sets for ``FAVICONS`` (Issue #1674)
 * Posts/Pages that use post-list will never be up to date (Issue #1671)
 * Support using post.text() in post_list_directive.tmpl (Issue #1671)

docs/manual.txt

@@ -1229,7 +1229,7 @@ You can disable comments for a post by adding a "nocomments" metadata field to i
     #639).
 
 An alternative or complement to comments are annotations. Nikola integrates
-the annotation service provided by `AnnotateIt. <annotateit.org>`_
+the annotation service provided by `AnnotateIt. <http://annotateit.org/>`_
 To use it, set the ``ANNOTATIONS`` option to True. This is specially useful
 if you want feedback on specific parts of your writing.
 
@@ -1273,7 +1273,7 @@ The ``conf.py`` options affecting images and gallery pages are these::
     # destination folder. A thumbnail will also be created in the same folder with
     # ``.thumbnail`` inserted in the file name before the file extension
     # (e.g. ``tesla.thumbnail.jpg``).
-    IMAGE_FOLDERS = {'images': ''}
+    IMAGE_FOLDERS = {'images': 'images'}
     # More image/gallery options:
     THUMBNAIL_SIZE = 180
     IMAGE_THUMBNAIL_SIZE = 400
@@ -1347,6 +1347,18 @@ The currently available filters are:
         ".html": [apply_to_text_file(string.upper)]
       }
 
+html_tidy_nowrap
+   Prettify HTML 5 documents with `tidy5 <http://www.html-tidy.org/>`_
+
+html_tidy_wrap
+   Prettify HTML 5 documents wrapped at 80 characters with `tidy5 <http://www.html-tidy.org/>`_
+
+html_tidy_wrap_attr
+   Prettify HTML 5 documents and wrap lines and attributes with `tidy5 <http://www.html-tidy.org/>`_
+
+html_tidy_mini
+   Minify HTML 5 into smaller documents with `tidy5 <http://www.html-tidy.org/>`_
+
 minify_lines
    **THIS FILTER HAS BEEN TURNED INTO A NOOP** and currently does nothing.
 

dodo.py

@@ -38,7 +38,7 @@ def set_nikola_test_locales():
                     lang = line.split('_')[0]
                     if lang not in languages:
                         try:
-                            locale.setlocale(locale.LC_ALL, line)
+                            locale.setlocale(locale.LC_ALL, str(line))
                         except:
                             continue
                         languages.add(lang)

nikola/filters.py

@@ -147,6 +147,31 @@ def jpegoptim(infile):
     return runinplace(r"jpegoptim -p --strip-all -q %1", infile)
 
 
+def html_tidy_nowrap(infile):
+    return _html_tidy_runner(infile, r"-quiet --show-info no --show-warnings no -utf8 -indent --indent-attributes no --sort-attributes alpha --wrap 0 --wrap-sections no --tidy-mark no -modify %1")
+
+
+def html_tidy_wrap(infile):
+    return _html_tidy_runner(infile, r"-quiet --show-info no --show-warnings no -utf8 -indent --indent-attributes no --sort-attributes alpha --wrap 80 --wrap-sections no --tidy-mark no -modify %1")
+
+
+def html_tidy_wrap_attr(infile):
+    return _html_tidy_runner(infile, r"-quiet --show-info no --show-warnings no -utf8 -indent --indent-attributes yes --sort-attributes alpha --wrap 80 --wrap-sections no --tidy-mark no -modify %1")
+
+
+def html_tidy_mini(infile):
+    return _html_tidy_runner(infile, r"-quiet --show-info no --show-warnings no -utf8 --indent-attributes no --sort-attributes alpha --wrap 0 --wrap-sections no --tidy-mark no -modify %1")
+
+
+def _html_tidy_runner(infile, options):
+    """ Warnings (returncode 1) are not critical, and *everything* is a warning """
+    try:
+        status = runinplace(r"tidy5 " + options, infile)
+    except subprocess.CalledProcessError as err:
+        status = 0 if err.returncode == 1 else err.returncode
+    return status
+
+
 @apply_to_text_file
 def minify_lines(data):
     return data

nikola/nikola.py

@@ -287,6 +287,7 @@ def __init__(self, **config):
         self.quiet = config.pop('__quiet__', False)
         self.configuration_filename = config.pop('__configuration_filename__', False)
         self.configured = bool(config)
+        self.injected_deps = defaultdict(list)
 
         self.rst_transforms = []
         self.template_hooks = {
@@ -410,7 +411,7 @@ def __init__(self, **config):
             'SHOW_UNTRANSLATED_POSTS': True,
             'SLUG_TAG_PATH': True,
             'SOCIAL_BUTTONS_CODE': '',
-            'SITE_URL': 'http://getnikola.com/',
+            'SITE_URL': 'https://example.com/',
             'STORY_INDEX': False,
             'STRIP_INDEXES': False,
             'SITEMAP_INCLUDE_FILELESS_DIRS': True,
@@ -1328,6 +1329,9 @@ def flatten(task):
             for task in flatten(pluginInfo.plugin_object.gen_tasks()):
                 assert 'basename' in task
                 task = self.clean_task_paths(task)
+                if 'task_dep' not in task:
+                    task['task_dep'] = []
+                task['task_dep'].extend(self.injected_deps[task['basename']])
                 yield task
                 for multi in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"):
                     flag = False

nikola/plugin_categories.py

@@ -75,6 +75,10 @@ def inject_templates(self):
             # so let’s just ignore it and be done with it.
             pass
 
+    def inject_dependency(self, target, dependency):
+        """Add 'dependency' to the target task's task_deps"""
+        self.site.injected_deps[target].append(dependency)
+
 
 class Command(BasePlugin, DoitCommand):
     """These plugins are exposed via the command line.

nikola/plugins/command/check.py

@@ -217,7 +217,7 @@ def analyze(self, task, find_sources=False):
                     fs_rel_path = fs_relpath_from_url_path(url_rel_path)
                     target_filename = os.path.join(self.site.config['OUTPUT_FOLDER'], fs_rel_path)
 
-                if any(re.match(x, target_filename) for x in self.whitelist):
+                if any(re.search(x, target_filename) for x in self.whitelist):
                     continue
                 elif target_filename not in self.existing_targets:
                     if os.path.exists(target_filename):
@@ -244,7 +244,7 @@ def scan_links(self, find_sources=False):
             if task.split(':')[0] in (
                     'render_tags', 'render_archive',
                     'render_galleries', 'render_indexes',
-                    'render_pages'
+                    'render_pages', 'render_posts',
                     'render_site') and '.html' in task:
                 if self.analyze(task, find_sources):
                     failure = True

nikola/plugins/command/init.py

@@ -50,7 +50,7 @@
 SAMPLE_CONF = {
     'BLOG_AUTHOR': "Your Name",
     'BLOG_TITLE': "Demo Site",
-    'SITE_URL': "http://getnikola.com/",
+    'SITE_URL': "https://example.com/",
     'BLOG_EMAIL': "joe@demo.site",
     'BLOG_DESCRIPTION': "This is a demo site for Nikola.",
     'DEFAULT_LANG': "en",
@@ -262,7 +262,7 @@ def create_empty_site(cls, target):
     def ask_questions(target):
         """Ask some questions about Nikola."""
         def urlhandler(default, toconf):
-            answer = ask('Site URL', 'http://getnikola.com/')
+            answer = ask('Site URL', 'https://example.com/')
             try:
                 answer = answer.decode('utf-8')
             except (AttributeError, UnicodeDecodeError):

tests/data/translated_titles/conf.py

@@ -23,7 +23,7 @@
 BLOG_TITLE = "Demo Site"  # (translatable)
 # This is the main URL for your site. It will be used
 # in a prominent link
-SITE_URL = "http://getnikola.com/"
+SITE_URL = "https://example.com/"
 # This is the URL where nikola's output will be deployed.
 # If not set, defaults to SITE_URL
 # BASE_URL = "http://getnikola.com/"

tests/test_integration.py

@@ -116,12 +116,12 @@ def fill_site(self):
     def test_index_in_sitemap(self):
         sitemap_path = os.path.join(self.target_dir, "output", "sitemap.xml")
         sitemap_data = io.open(sitemap_path, "r", encoding="utf8").read()
-        self.assertTrue('<loc>http://getnikola.com/index.html</loc>' in sitemap_data)
+        self.assertTrue('<loc>https://example.com/index.html</loc>' in sitemap_data)
 
     def test_avoid_double_slash_in_rss(self):
         rss_path = os.path.join(self.target_dir, "output", "rss.xml")
         rss_data = io.open(rss_path, "r", encoding="utf8").read()
-        self.assertFalse('http://getnikola.com//' in rss_data)
+        self.assertFalse('https://example.com//' in rss_data)
 
 
 class RepeatedPostsSetting(DemoBuildTest):
@@ -272,8 +272,8 @@ def patch_site(self):
         conf_path = os.path.join(self.target_dir, "conf.py")
         with io.open(conf_path, "r", encoding="utf-8") as inf:
             data = inf.read()
-            data = data.replace('SITE_URL = "http://getnikola.com/"',
-                                'SITE_URL = "http://getnikola.com/foo/bar/"')
+            data = data.replace('SITE_URL = "https://example.com/"',
+                                'SITE_URL = "https://example.com/foo/bar/"')
         with io.open(conf_path, "w+", encoding="utf8") as outf:
             outf.write(data)
 
@@ -295,8 +295,8 @@ def test_index_in_sitemap(self):
         """Test that the correct path is in sitemap, and not the wrong one."""
         sitemap_path = os.path.join(self.target_dir, "output", "sitemap.xml")
         sitemap_data = io.open(sitemap_path, "r", encoding="utf8").read()
-        self.assertFalse('<loc>http://getnikola.com/</loc>' in sitemap_data)
-        self.assertTrue('<loc>http://getnikola.com/foo/bar/index.html</loc>' in sitemap_data)
+        self.assertFalse('<loc>https://example.com/</loc>' in sitemap_data)
+        self.assertTrue('<loc>https://example.com/foo/bar/index.html</loc>' in sitemap_data)
 
 
 class TestCheck(DemoBuildTest):
@@ -321,16 +321,16 @@ class TestCheckAbsoluteSubFolder(TestCheck):
     """Validate links in a site which is:
 
     * built in URL_TYPE="absolute"
-    * deployable to a subfolder (BASE_URL="http://getnikola.com/foo/")
+    * deployable to a subfolder (BASE_URL="https://example.com/foo/")
     """
 
     @classmethod
     def patch_site(self):
         conf_path = os.path.join(self.target_dir, "conf.py")
         with io.open(conf_path, "r", encoding="utf-8") as inf:
             data = inf.read()
-            data = data.replace('SITE_URL = "http://getnikola.com/"',
-                                'SITE_URL = "http://getnikola.com/foo/"')
+            data = data.replace('SITE_URL = "https://example.com/"',
+                                'SITE_URL = "https://example.com/foo/"')
             data = data.replace("# URL_TYPE = 'rel_path'",
                                 "URL_TYPE = 'absolute'")
         with io.open(conf_path, "w+", encoding="utf8") as outf:
@@ -341,23 +341,23 @@ def test_index_in_sitemap(self):
         """Test that the correct path is in sitemap, and not the wrong one."""
         sitemap_path = os.path.join(self.target_dir, "output", "sitemap.xml")
         sitemap_data = io.open(sitemap_path, "r", encoding="utf8").read()
-        self.assertTrue('<loc>http://getnikola.com/foo/index.html</loc>' in sitemap_data)
+        self.assertTrue('<loc>https://example.com/foo/index.html</loc>' in sitemap_data)
 
 
 class TestCheckFullPathSubFolder(TestCheckAbsoluteSubFolder):
     """Validate links in a site which is:
 
     * built in URL_TYPE="full_path"
-    * deployable to a subfolder (BASE_URL="http://getnikola.com/foo/")
+    * deployable to a subfolder (BASE_URL="https://example.com/foo/")
     """
 
     @classmethod
     def patch_site(self):
         conf_path = os.path.join(self.target_dir, "conf.py")
         with io.open(conf_path, "r", encoding="utf-8") as inf:
             data = inf.read()
-            data = data.replace('SITE_URL = "http://getnikola.com/"',
-                                'SITE_URL = "http://getnikola.com/foo/"')
+            data = data.replace('SITE_URL = "https://example.com/"',
+                                'SITE_URL = "https://example.com/foo/"')
             data = data.replace("# URL_TYPE = 'rel_path'",
                                 "URL_TYPE = 'full_path'")
         with io.open(conf_path, "w+", encoding="utf8") as outf:
@@ -423,8 +423,8 @@ def test_index_in_sitemap(self):
         """Test that the correct path is in sitemap, and not the wrong one."""
         sitemap_path = os.path.join(self.target_dir, "output", "sitemap.xml")
         sitemap_data = io.open(sitemap_path, "r", encoding="utf8").read()
-        self.assertFalse('<loc>http://getnikola.com/</loc>' in sitemap_data)
-        self.assertTrue('<loc>http://getnikola.com/blog/index.html</loc>' in sitemap_data)
+        self.assertFalse('<loc>https://example.com/</loc>' in sitemap_data)
+        self.assertTrue('<loc>https://example.com/blog/index.html</loc>' in sitemap_data)
 
 
 class MonthlyArchiveTest(DemoBuildTest):