Merge branch 'master' into fix-1772

CHANGES.txt

@@ -14,6 +14,7 @@ Features
 Bugfixes
 --------
 
+* Extract ``nikola check`` target list from actual task list instead of parsing (Issue #1758)
 * Treat special-purpose “draft” tag case-insensitive
 * Avoid some rebuild loops (Issue #1747)
 * Better error if two posts/pages output conflict (Issue #1749)

nikola/plugins/command/check.py

@@ -25,6 +25,7 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 from __future__ import print_function
+from collections import defaultdict
 import os
 import re
 import sys
@@ -34,6 +35,7 @@
 except ImportError:
     from urllib.parse import unquote, urlparse, urljoin, urldefrag  # NOQA
 
+from doit.loader import generate_tasks
 import lxml.html
 try:
     import requests
@@ -44,33 +46,28 @@
 from nikola.utils import get_logger, req_missing
 
 
-def _call_nikola_list(l, site, arguments):
-    class NotReallyAStream(object):
-        """A massive hack."""
-        out = []
+def _call_nikola_list(site):
+    files = []
+    deps = defaultdict(list)
+    for task in generate_tasks('render_site', site.gen_tasks('render_site', "Task", '')):
+        files.extend(task.targets)
+        for target in task.targets:
+            deps[target].extend(task.file_dep)
+    for task in generate_tasks('post_render', site.gen_tasks('render_site', "LateTask", '')):
+        files.extend(task.targets)
+        for target in task.targets:
+            deps[target].extend(task.file_dep)
+    return files, deps
 
-        def write(self, t):
-            self.out.append(t)
 
-    oldstream = l.outstream
-    newstream = NotReallyAStream()
-    try:
-        l.outstream = newstream
-        l.parse_execute(arguments)
-        return newstream.out
-    finally:
-        l.outstream = oldstream
-
-
-def real_scan_files(l, site):
+def real_scan_files(site):
     task_fnames = set([])
     real_fnames = set([])
     output_folder = site.config['OUTPUT_FOLDER']
     # First check that all targets are generated in the right places
-    for task in _call_nikola_list(l, site, ["--all"]):
-        task = task.strip()
-        if output_folder in task and ':' in task:
-            fname = task.split(':', 1)[-1]
+    for fname in _call_nikola_list(site)[0]:
+        fname = fname.strip()
+        if fname.startswith(output_folder):
             task_fnames.add(fname)
     # And now check that there are no non-target files
     for root, dirs, files in os.walk(output_folder, followlinks=True):
@@ -154,7 +151,6 @@ class CommandCheck(Command):
     def _execute(self, options, args):
         """Check the generated site."""
         self.logger = get_logger('check', self.site.loghandlers)
-        self.l = self._doitargs['cmds'].get_plugin('list')(config=self.config, **self._doitargs)
 
         if not options['links'] and not options['files'] and not options['clean']:
             print(self.help())
@@ -175,21 +171,25 @@ def _execute(self, options, args):
     existing_targets = set([])
     checked_remote_targets = {}
 
-    def analyze(self, task, find_sources=False, check_remote=False):
+    def analyze(self, fname, find_sources=False, check_remote=False):
         rv = False
         self.whitelist = [re.compile(x) for x in self.site.config['LINK_CHECK_WHITELIST']]
         base_url = urlparse(self.site.config['BASE_URL'])
         self.existing_targets.add(self.site.config['SITE_URL'])
         self.existing_targets.add(self.site.config['BASE_URL'])
         url_type = self.site.config['URL_TYPE']
 
+        deps = {}
+        if find_sources:
+            deps = _call_nikola_list(self.site)[1]
+
         if check_remote and requests is None:
             req_missing(['requests'], 'check remote links')
 
         if url_type in ('absolute', 'full_path'):
             url_netloc_to_root = urlparse(self.site.config['BASE_URL']).path
         try:
-            filename = task.split(":")[-1]
+            filename = fname
 
             if filename.startswith(self.site.config['CACHE_FOLDER']):
                 # Do not look at links in the cache, which are not parsed by
@@ -262,7 +262,7 @@ def analyze(self, task, find_sources=False, check_remote=False):
                         self.logger.warn("Broken link in {0}: {1}".format(filename, target))
                         if find_sources:
                             self.logger.warn("Possible sources:")
-                            self.logger.warn("\n".join(_call_nikola_list(self.l, self.site, ["--deps", task])))
+                            self.logger.warn("\n".join(deps[filename]))
                             self.logger.warn("===============================\n")
         except Exception as exc:
             self.logger.error("Error with: {0} {1}".format(filename, exc))
@@ -273,14 +273,11 @@ def scan_links(self, find_sources=False, check_remote=False):
         self.logger.info("===============\n")
         self.logger.notice("{0} mode".format(self.site.config['URL_TYPE']))
         failure = False
-        for task in _call_nikola_list(self.l, self.site, ["--all"]):
-            task = task.strip()
-            if task.split(':')[0] in (
-                    'render_tags', 'render_archive',
-                    'render_galleries', 'render_indexes',
-                    'render_pages', 'render_posts',
-                    'render_site') and '.html' in task:
-                if self.analyze(task, find_sources, check_remote):
+        # Maybe we should just examine all HTML files
+        output_folder = self.site.config['OUTPUT_FOLDER']
+        for fname in _call_nikola_list(self.site)[0]:
+            if fname.startswith(output_folder) and '.html' == fname[-5:]:
+                if self.analyze(fname, find_sources, check_remote):
                     failure = True
         if not failure:
             self.logger.info("All links checked.")
@@ -290,7 +287,7 @@ def scan_files(self):
         failure = False
         self.logger.info("Checking Files:")
         self.logger.info("===============\n")
-        only_on_output, only_on_input = real_scan_files(self.l, self.site)
+        only_on_output, only_on_input = real_scan_files(self.site)
 
         # Ignore folders
         only_on_output = [p for p in only_on_output if not os.path.isdir(p)]
@@ -312,7 +309,7 @@ def scan_files(self):
         return failure
 
     def clean_files(self):
-        only_on_output, _ = real_scan_files(self.l, self.site)
+        only_on_output, _ = real_scan_files(self.site)
         for f in only_on_output:
             os.unlink(f)
         return True

nikola/plugins/command/github_deploy.py

@@ -86,8 +86,7 @@ def _execute(self, command, args):
             sys.exit(build)
 
         # Clean non-target files
-        l = self._doitargs['cmds'].get_plugin('list')(config=self.config, **self._doitargs)
-        only_on_output, _ = real_scan_files(l, self.site)
+        only_on_output, _ = real_scan_files(self.site)
         for f in only_on_output:
             os.unlink(f)
 

nikola/plugins/command/orphans.py

@@ -41,6 +41,5 @@ class CommandOrphans(Command):
 Output contains filenames only (it is passable to `xargs rm` or the like)."""
 
     def _execute(self, options, args):
-        l = self._doitargs['cmds'].get_plugin('list')(config=self.config, **self._doitargs)
-        orphans = real_scan_files(l, self.site)[0]
+        orphans = real_scan_files(self.site)[0]
         print('\n'.join([p for p in orphans if not os.path.isdir(p)]))

nikola/plugins/command/status.py

@@ -40,8 +40,26 @@ class CommandDeploy(Command):
     doc_purpose = "display site status"
     doc_description = "Show information about the posts and site deployment."
     logger = None
-
-    def _execute(self, command, args):
+    cmd_options = [
+        {
+            'name': 'list_drafts',
+            'short': 'd',
+            'long': 'list-drafts',
+            'type': bool,
+            'default': False,
+            'help': 'List all drafts',
+        },
+        {
+            'name': 'list_scheduled',
+            'short': 's',
+            'long': 'list-scheduled',
+            'type': bool,
+            'default': False,
+            'help': 'List all scheduled posts',
+        },
+    ]
+
+    def _execute(self, options, args):
 
         self.site.scan_posts()
 
@@ -73,23 +91,29 @@ def _execute(self, command, args):
             else:
                 print("Last deployment {0} ago.".format(self.human_time(last_deploy_offset)))
 
+        now = datetime.utcnow().replace(tzinfo=gettz("UTC"))
+
         posts_count = len(self.site.all_posts)
-        posts_drafts = 0
-        posts_scheduled = 0
-        nearest_scheduled_offset = None
-
-        for post in self.site.all_posts:
-            if post.is_draft:
-                posts_drafts = posts_drafts + 1
-            if post.publish_later:
-                posts_scheduled = posts_scheduled + 1
-                post_due_offset = post.date - datetime.utcnow().replace(tzinfo=gettz("UTC"))
-                if (nearest_scheduled_offset is None) or (post_due_offset.seconds < nearest_scheduled_offset.seconds):
-                    nearest_scheduled_offset = post_due_offset
-
-        if posts_scheduled > 0 and nearest_scheduled_offset is not None:
-            print("{0} to next scheduled post.".format(self.human_time(nearest_scheduled_offset)))
-        print("{0} posts in total, {1} scheduled, and {2} drafts.".format(posts_count, posts_scheduled, posts_drafts))
+
+        # find all drafts
+        posts_drafts = [post for post in self.site.all_posts if post.is_draft]
+        posts_drafts = sorted(posts_drafts, key=lambda post: post.source_path)
+
+        # find all scheduled posts with offset from now until publishing time
+        posts_scheduled = [(post.date - now, post) for post in self.site.all_posts if post.publish_later]
+        posts_scheduled = sorted(posts_scheduled, key=lambda offset_post: (offset_post[0], offset_post[1].source_path))
+
+        if len(posts_scheduled) > 0:
+            if options['list_scheduled']:
+                for offset, post in posts_scheduled:
+                    print("Scheduled: '{1}' ({2}; source: {3}) in {0}".format(self.human_time(offset), post.meta('title'), post.permalink(), post.source_path))
+            else:
+                offset, post = posts_scheduled[0]
+                print("{0} to next scheduled post ('{1}'; {2}; source: {3}).".format(self.human_time(offset), post.meta('title'), post.permalink(), post.source_path))
+        if options['list_drafts']:
+            for post in posts_drafts:
+                print("Draft: '{0}' ({1}; source: {2})".format(post.meta('title'), post.permalink(), post.source_path))
+        print("{0} posts in total, {1} scheduled, and {2} drafts.".format(posts_count, len(posts_scheduled), len(posts_drafts)))
 
     def human_time(self, dt):
         days = dt.days

nikola/plugins/task/listings.py

@@ -133,7 +133,7 @@ def render_listing(in_name, out_name, input_folder, output_folder, folders=[], f
                         os.path.join(
                             self.kw['output_folder'],
                             output_folder))))
-            if self.site.config['COPY_SOURCES']:
+            if self.site.config['COPY_SOURCES'] and in_name:
                 source_link = permalink[:-5]  # remove '.html'
             else:
                 source_link = None