From bc60ed9942e8df2011acc92889eaea41a31d1c03 Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Fri, 8 May 2015 17:27:08 -0300
Subject: [PATCH 01/21] moved scan_posts dumbly into a plugin

---
 CHANGES.txt                           |   1 +
 nikola/nikola.py                      | 127 +------------------
 nikola/plugins/task/scan_posts.plugin |  10 ++
 nikola/plugins/task/scan_posts.py     | 174 ++++++++++++++++++++++++++
 4 files changed, 191 insertions(+), 121 deletions(-)
 create mode 100644 nikola/plugins/task/scan_posts.plugin
 create mode 100644 nikola/plugins/task/scan_posts.py

diff --git a/CHANGES.txt b/CHANGES.txt
index ac5209fba3..28bbd9611c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -10,6 +10,7 @@ Features
 Bugfixes
 --------
 
+* Scanning of posts refactored out of core (Issue #1700)
 * Handle strange URLs, like ed2k:// (Issue #1695)
 * Fix very old metadata format support (Issue #1689)
 
diff --git a/nikola/nikola.py b/nikola/nikola.py
index 8dbde841ff..8b2b0006bb 100644
--- a/nikola/nikola.py
+++ b/nikola/nikola.py
@@ -1352,129 +1352,14 @@ def flatten(task):
             'task_dep': task_dep
         }
 
-    def scan_posts(self, really=False, ignore_quit=False, quiet=False):
+    def scan_posts(self):
         """Scan all the posts."""
-        if self._scanned and not really:
+        # FIXME this is temporary while moving things out to a plugin
+        # Why doesn't getPluginByName work????
+        if self._scanned:
             return
-
-        self.global_data = {}
-        self.posts = []
-        self.all_posts = []
-        self.posts_per_year = defaultdict(list)
-        self.posts_per_month = defaultdict(list)
-        self.posts_per_tag = defaultdict(list)
-        self.posts_per_category = defaultdict(list)
-        self.post_per_file = {}
-        self.timeline = []
-        self.pages = []
-
-        seen = set([])
-        if not self.quiet and not quiet:
-            print("Scanning posts", end='', file=sys.stderr)
-        slugged_tags = set([])
-        quit = False
-        for wildcard, destination, template_name, use_in_feeds in \
-                self.config['post_pages']:
-            if not self.quiet and not quiet:
-                print(".", end='', file=sys.stderr)
-            dirname = os.path.dirname(wildcard)
-            for dirpath, _, _ in os.walk(dirname, followlinks=True):
-                dest_dir = os.path.normpath(os.path.join(destination,
-                                            os.path.relpath(dirpath, dirname)))  # output/destination/foo/
-                # Get all the untranslated paths
-                dir_glob = os.path.join(dirpath, os.path.basename(wildcard))  # posts/foo/*.rst
-                untranslated = glob.glob(dir_glob)
-                # And now get all the translated paths
-                translated = set([])
-                for lang in self.config['TRANSLATIONS'].keys():
-                    if lang == self.config['DEFAULT_LANG']:
-                        continue
-                    lang_glob = utils.get_translation_candidate(self.config, dir_glob, lang)  # posts/foo/*.LANG.rst
-                    translated = translated.union(set(glob.glob(lang_glob)))
-                # untranslated globs like *.rst often match translated paths too, so remove them
-                # and ensure x.rst is not in the translated set
-                untranslated = set(untranslated) - translated
-
-                # also remove from translated paths that are translations of
-                # paths in untranslated_list, so x.es.rst is not in the untranslated set
-                for p in untranslated:
-                    translated = translated - set([utils.get_translation_candidate(self.config, p, l) for l in self.config['TRANSLATIONS'].keys()])
-
-                full_list = list(translated) + list(untranslated)
-                # We eliminate from the list the files inside any .ipynb folder
-                full_list = [p for p in full_list
-                             if not any([x.startswith('.')
-                                         for x in p.split(os.sep)])]
-
-                for base_path in full_list:
-                    if base_path in seen:
-                        continue
-                    else:
-                        seen.add(base_path)
-                    post = Post(
-                        base_path,
-                        self.config,
-                        dest_dir,
-                        use_in_feeds,
-                        self.MESSAGES,
-                        template_name,
-                        self.get_compiler(base_path)
-                    )
-                    self.timeline.append(post)
-                    self.global_data[post.source_path] = post
-                    if post.use_in_feeds:
-                        self.posts.append(post)
-                        self.posts_per_year[
-                            str(post.date.year)].append(post)
-                        self.posts_per_month[
-                            '{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post)
-                        for tag in post.alltags:
-                            _tag_slugified = utils.slugify(tag)
-                            if _tag_slugified in slugged_tags:
-                                if tag not in self.posts_per_tag:
-                                    # Tags that differ only in case
-                                    other_tag = [existing for existing in self.posts_per_tag.keys() if utils.slugify(existing) == _tag_slugified][0]
-                                    utils.LOGGER.error('You have tags that are too similar: {0} and {1}'.format(tag, other_tag))
-                                    utils.LOGGER.error('Tag {0} is used in: {1}'.format(tag, post.source_path))
-                                    utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join([p.source_path for p in self.posts_per_tag[other_tag]])))
-                                    quit = True
-                            else:
-                                slugged_tags.add(utils.slugify(tag, force=True))
-                            self.posts_per_tag[tag].append(post)
-                        self.posts_per_category[post.meta('category')].append(post)
-
-                    if post.is_post:
-                        # unpublished posts
-                        self.all_posts.append(post)
-                    else:
-                        self.pages.append(post)
-
-                    for lang in self.config['TRANSLATIONS'].keys():
-                        self.post_per_file[post.destination_path(lang=lang)] = post
-                        self.post_per_file[post.destination_path(lang=lang, extension=post.source_ext())] = post
-
-        # Sort everything.
-        self.timeline.sort(key=lambda p: p.date)
-        self.timeline.reverse()
-        self.posts.sort(key=lambda p: p.date)
-        self.posts.reverse()
-        self.all_posts.sort(key=lambda p: p.date)
-        self.all_posts.reverse()
-        self.pages.sort(key=lambda p: p.date)
-        self.pages.reverse()
-
-        for i, p in enumerate(self.posts[1:]):
-            p.next_post = self.posts[i]
-        for i, p in enumerate(self.posts[:-1]):
-            p.prev_post = self.posts[i + 1]
-        self._scanned = True
-        if not self.quiet and not quiet:
-            print("done!", file=sys.stderr)
-
-        signal('scanned').send(self)
-
-        if quit and not ignore_quit:
-            sys.exit(1)
+        p = [p for p in self.plugin_manager.getPluginsOfCategory('Task') if p.name == 'scan_posts'][0]
+        list(p.plugin_object.gen_tasks())
 
     def generic_page_renderer(self, lang, post, filters):
         """Render post fragments to final HTML pages."""
diff --git a/nikola/plugins/task/scan_posts.plugin b/nikola/plugins/task/scan_posts.plugin
new file mode 100644
index 0000000000..6d2351f59e
--- /dev/null
+++ b/nikola/plugins/task/scan_posts.plugin
@@ -0,0 +1,10 @@
+[Core]
+Name = scan_posts
+Module = scan_posts
+
+[Documentation]
+Author = Roberto Alsina
+Version = 1.0
+Website = http://getnikola.com
+Description = Scan posts and create timeline
+
diff --git a/nikola/plugins/task/scan_posts.py b/nikola/plugins/task/scan_posts.py
new file mode 100644
index 0000000000..43da874bb2
--- /dev/null
+++ b/nikola/plugins/task/scan_posts.py
@@ -0,0 +1,174 @@
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012-2015 Roberto Alsina and others.
+
+# Permission is hereby granted, free of charge, to any
+# person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the
+# Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the
+# Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice
+# shall be included in all copies or substantial portions of
+# the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+from __future__ import unicode_literals, print_function
+from collections import defaultdict
+import glob
+import os
+import sys
+
+from blinker import signal
+
+from nikola.plugin_categories import Task
+from nikola import utils
+from nikola.post import Post
+
+
+class ScanPosts(Task):
+    """Render pages into output."""
+
+    name = "scan_posts"
+
+    def gen_tasks(self):
+        """Build final pages from metadata and HTML fragments."""
+        kw = {
+            "post_pages": self.site.config["post_pages"],
+            "translations": self.site.config["TRANSLATIONS"],
+            "filters": self.site.config["FILTERS"],
+            "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'],
+            "demote_headers": self.site.config['DEMOTE_HEADERS'],
+        }
+        self.site.global_data = {}
+        self.site.posts = []
+        self.site.all_posts = []
+        self.site.posts_per_year = defaultdict(list)
+        self.site.posts_per_month = defaultdict(list)
+        self.site.posts_per_tag = defaultdict(list)
+        self.site.posts_per_category = defaultdict(list)
+        self.site.post_per_file = {}
+        self.site.timeline = []
+        self.site.pages = []
+
+        seen = set([])
+        if not self.site.quiet:
+            print("Scanning posts", end='', file=sys.stderr)
+
+        slugged_tags = set([])
+        quit = False
+        for wildcard, destination, template_name, use_in_feeds in \
+                self.site.config['post_pages']:
+            if not self.site.quiet:
+                print(".", end='', file=sys.stderr)
+            dirname = os.path.dirname(wildcard)
+            for dirpath, _, _ in os.walk(dirname, followlinks=True):
+                dest_dir = os.path.normpath(os.path.join(destination,
+                                            os.path.relpath(dirpath, dirname)))  # output/destination/foo/
+                # Get all the untranslated paths
+                dir_glob = os.path.join(dirpath, os.path.basename(wildcard))  # posts/foo/*.rst
+                untranslated = glob.glob(dir_glob)
+                # And now get all the translated paths
+                translated = set([])
+                for lang in self.site.config['TRANSLATIONS'].keys():
+                    if lang == self.site.config['DEFAULT_LANG']:
+                        continue
+                    lang_glob = utils.get_translation_candidate(self.site.config, dir_glob, lang)  # posts/foo/*.LANG.rst
+                    translated = translated.union(set(glob.glob(lang_glob)))
+                # untranslated globs like *.rst often match translated paths too, so remove them
+                # and ensure x.rst is not in the translated set
+                untranslated = set(untranslated) - translated
+
+                # also remove from translated paths that are translations of
+                # paths in untranslated_list, so x.es.rst is not in the untranslated set
+                for p in untranslated:
+                    translated = translated - set([utils.get_translation_candidate(self.site.config, p, l) for l in self.site.config['TRANSLATIONS'].keys()])
+
+                full_list = list(translated) + list(untranslated)
+                # We eliminate from the list the files inside any .ipynb folder
+                full_list = [p for p in full_list
+                             if not any([x.startswith('.')
+                                         for x in p.split(os.sep)])]
+
+                for base_path in full_list:
+                    if base_path in seen:
+                        continue
+                    else:
+                        seen.add(base_path)
+                    post = Post(
+                        base_path,
+                        self.site.config,
+                        dest_dir,
+                        use_in_feeds,
+                        self.site.MESSAGES,
+                        template_name,
+                        self.site.get_compiler(base_path)
+                    )
+                    self.site.timeline.append(post)
+                    self.site.global_data[post.source_path] = post
+                    if post.use_in_feeds:
+                        self.site.posts.append(post)
+                        self.site.posts_per_year[
+                            str(post.date.year)].append(post)
+                        self.site.posts_per_month[
+                            '{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post)
+                        for tag in post.alltags:
+                            _tag_slugified = utils.slugify(tag)
+                            if _tag_slugified in slugged_tags:
+                                if tag not in self.site.posts_per_tag:
+                                    # Tags that differ only in case
+                                    other_tag = [existing for existing in self.site.posts_per_tag.keys() if utils.slugify(existing) == _tag_slugified][0]
+                                    utils.LOGGER.error('You have tags that are too similar: {0} and {1}'.format(tag, other_tag))
+                                    utils.LOGGER.error('Tag {0} is used in: {1}'.format(tag, post.source_path))
+                                    utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join([p.source_path for p in self.site.posts_per_tag[other_tag]])))
+                                    quit = True
+                            else:
+                                slugged_tags.add(utils.slugify(tag, force=True))
+                            self.site.posts_per_tag[tag].append(post)
+                        self.site.posts_per_category[post.meta('category')].append(post)
+
+                    if post.is_post:
+                        # unpublished posts
+                        self.site.all_posts.append(post)
+                    else:
+                        self.site.pages.append(post)
+
+                    for lang in self.site.config['TRANSLATIONS'].keys():
+                        self.site.post_per_file[post.destination_path(lang=lang)] = post
+                        self.site.post_per_file[post.destination_path(lang=lang, extension=post.source_ext())] = post
+
+        # Sort everything.
+        self.site.timeline.sort(key=lambda p: p.date)
+        self.site.timeline.reverse()
+        self.site.posts.sort(key=lambda p: p.date)
+        self.site.posts.reverse()
+        self.site.all_posts.sort(key=lambda p: p.date)
+        self.site.all_posts.reverse()
+        self.site.pages.sort(key=lambda p: p.date)
+        self.site.pages.reverse()
+
+        for i, p in enumerate(self.site.posts[1:]):
+            p.next_post = self.site.posts[i]
+        for i, p in enumerate(self.site.posts[:-1]):
+            p.prev_post = self.site.posts[i + 1]
+        self.site._scanned = True
+        if not self.site.quiet:
+            print("done!", file=sys.stderr)
+
+        signal('scanned').send(self)
+
+        if quit and not ignore_quit:
+            sys.exit(1)
+
+        yield self.group_task()

From 94ed9f7e1495a8eab6b587404b4b2039ff7d24ac Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Fri, 8 May 2015 17:34:01 -0300
Subject: [PATCH 02/21] baby steps

---
 docs/internals.txt | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/docs/internals.txt b/docs/internals.txt
index 2ade3c4be1..9de56dff2c 100644
--- a/docs/internals.txt
+++ b/docs/internals.txt
@@ -95,17 +95,14 @@ posts are added into RSS feeds and stories are not. All of them are in a list ca
 "the timeline" formed by objects of class ``Post``.
 
 When you are creating a task that needs the list of posts and/or stories (for example,
-the RSS creation plugin), your plugin should call ``self.site.scan_posts()`` to ensure
-the timeline is created and available in ``self.site.timeline``. You should not modify
-the timeline, because it will cause consistency issues.
+the RSS creation plugin) on task execution time, your plugin should have a dependency 
+on the ``scan_posts`` task to ensure the timeline is created and available in 
+``self.site.timeline``. You should not modify the timeline, because it will cause consistency issues.
 
 .. sidebar:: scan_posts
 
-   The scan_posts function is what reads your site and creates the timeline.
-
-   I am considering moving scan_posts off the core and into its own plugin
-   so it can be replaced (for example, by a version that reads a database
-   instead of scanning a folder tree).
+   The ``Nikola.scan_posts`` function can be used in plugins to force the
+   timeline creation, for example, while creating the tasks.
 
 Your plugin can use the timeline to generate "stuff" (technical term). For example,
 Nikola comes with plugins that use the timeline to create a website (surprised?).

From 597732df06714176c1e54a7692829aac4b8f9e64 Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Sat, 9 May 2015 12:09:29 -0300
Subject: [PATCH 03/21] stepstep

---
 nikola/nikola.py                  | 4 ++--
 nikola/plugin_categories.py       | 7 +++++++
 nikola/plugins/task/scan_posts.py | 8 ++++----
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/nikola/nikola.py b/nikola/nikola.py
index 8b2b0006bb..194c836bb6 100644
--- a/nikola/nikola.py
+++ b/nikola/nikola.py
@@ -1358,8 +1358,8 @@ def scan_posts(self):
         # Why doesn't getPluginByName work????
         if self._scanned:
             return
-        p = [p for p in self.plugin_manager.getPluginsOfCategory('Task') if p.name == 'scan_posts'][0]
-        list(p.plugin_object.gen_tasks())
+        for p in self.plugin_manager.getPluginsOfCategory('PostScanner'):
+            p.plugin_object.scan()
 
     def generic_page_renderer(self, lang, post, filters):
         """Render post fragments to final HTML pages."""
diff --git a/nikola/plugin_categories.py b/nikola/plugin_categories.py
index e880d46578..cb0b88c1e8 100644
--- a/nikola/plugin_categories.py
+++ b/nikola/plugin_categories.py
@@ -79,6 +79,13 @@ def inject_dependency(self, target, dependency):
         """Add 'dependency' to the target task's task_deps"""
         self.site.injected_deps[target].append(dependency)
 
+class PostScanner(BasePlugin):
+    """The scan method of these plugins is called by Nikola.scan_posts."""
+
+    def scan(self):
+        """Load posts into the timeline."""
+        raise NotImplementedError()
+
 
 class Command(BasePlugin, DoitCommand):
     """These plugins are exposed via the command line.
diff --git a/nikola/plugins/task/scan_posts.py b/nikola/plugins/task/scan_posts.py
index 43da874bb2..0c9dc81e59 100644
--- a/nikola/plugins/task/scan_posts.py
+++ b/nikola/plugins/task/scan_posts.py
@@ -32,17 +32,17 @@
 
 from blinker import signal
 
-from nikola.plugin_categories import Task
+from nikola.plugin_categories import PostScanner
 from nikola import utils
 from nikola.post import Post
 
 
-class ScanPosts(Task):
+class ScanPosts(PostScanner):
     """Render pages into output."""
 
     name = "scan_posts"
 
-    def gen_tasks(self):
+    def scan(self):
         """Build final pages from metadata and HTML fragments."""
         kw = {
             "post_pages": self.site.config["post_pages"],
@@ -73,7 +73,7 @@ def gen_tasks(self):
             if not self.site.quiet:
                 print(".", end='', file=sys.stderr)
             dirname = os.path.dirname(wildcard)
-            for dirpath, _, _ in os.walk(dirname, followlinks=True):
+            for dirpath, _, _ in os.walk(dirname, followlinks=True ):
                 dest_dir = os.path.normpath(os.path.join(destination,
                                             os.path.relpath(dirpath, dirname)))  # output/destination/foo/
                 # Get all the untranslated paths

From 2f7dd8ff3ba776ebbc4eb6e15f90d3d6ab2671b3 Mon Sep 17 00:00:00 2001
From: Chris Warrick <kwpolska@gmail.com>
Date: Sat, 9 May 2015 18:30:24 +0200
Subject: [PATCH 04/21] add PostScanner to categories

Signed-off-by: Chris Warrick <kwpolska@gmail.com>
---
 nikola/nikola.py            | 4 ++--
 nikola/plugin_categories.py | 5 ++++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/nikola/nikola.py b/nikola/nikola.py
index 194c836bb6..7f519c6125 100644
--- a/nikola/nikola.py
+++ b/nikola/nikola.py
@@ -30,7 +30,6 @@
 from copy import copy
 from pkg_resources import resource_filename
 import datetime
-import glob
 import locale
 import os
 import json
@@ -54,7 +53,6 @@
 from yapsy.PluginManager import PluginManager
 from blinker import signal
 
-from .post import Post
 from . import DEBUG, utils
 from .plugin_categories import (
     Command,
@@ -67,6 +65,7 @@
     TemplateSystem,
     SignalHandler,
     ConfigPlugin,
+    PostScanner,
 )
 
 if DEBUG:
@@ -667,6 +666,7 @@ def __init__(self, **config):
             "MarkdownExtension": MarkdownExtension,
             "SignalHandler": SignalHandler,
             "ConfigPlugin": ConfigPlugin,
+            "PostScanner": PostScanner,
         })
         self.plugin_manager.setPluginInfoExtension('plugin')
         extra_plugins_dirs = self.config['EXTRA_PLUGINS_DIRS']
diff --git a/nikola/plugin_categories.py b/nikola/plugin_categories.py
index cb0b88c1e8..51265d2022 100644
--- a/nikola/plugin_categories.py
+++ b/nikola/plugin_categories.py
@@ -42,7 +42,9 @@
     'Task',
     'TaskMultiplier',
     'TemplateSystem',
-    'SignalHandler'
+    'SignalHandler',
+    'ConfigPlugin',
+    'PostScanner',
 ]
 
 
@@ -79,6 +81,7 @@ def inject_dependency(self, target, dependency):
         """Add 'dependency' to the target task's task_deps"""
         self.site.injected_deps[target].append(dependency)
 
+
 class PostScanner(BasePlugin):
     """The scan method of these plugins is called by Nikola.scan_posts."""
 

From 835b8605810d637dc3fe9fa926c841a4176af7d5 Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Sat, 9 May 2015 16:30:25 -0300
Subject: [PATCH 05/21] more refactoring

---
 nikola/nikola.py                  | 75 ++++++++++++++++++++++++++++++-
 nikola/plugins/task/scan_posts.py | 73 +++---------------------------
 2 files changed, 79 insertions(+), 69 deletions(-)

diff --git a/nikola/nikola.py b/nikola/nikola.py
index 7f519c6125..72c2a011b4 100644
--- a/nikola/nikola.py
+++ b/nikola/nikola.py
@@ -1355,11 +1355,82 @@ def flatten(task):
     def scan_posts(self):
         """Scan all the posts."""
         # FIXME this is temporary while moving things out to a plugin
-        # Why doesn't getPluginByName work????
         if self._scanned:
             return
+
+        # Reset things
+        self.global_data = {}
+        self.posts = []
+        self.all_posts = []
+        self.posts_per_year = defaultdict(list)
+        self.posts_per_month = defaultdict(list)
+        self.posts_per_tag = defaultdict(list)
+        self.posts_per_category = defaultdict(list)
+        self.post_per_file = {}
+        self.timeline = []
+        self.pages = []
+
         for p in self.plugin_manager.getPluginsOfCategory('PostScanner'):
-            p.plugin_object.scan()
+            timeline, global_data = p.plugin_object.scan()
+            # FIXME: can there be conflicts here?
+            self.timeline.extend(timeline)
+            self.global_data.update(global_data)
+
+        slugged_tags = set([])
+        for post in self.timeline:
+            if post.use_in_feeds:
+                self.site.posts.append(post)
+                self.site.posts_per_year[
+                    str(post.date.year)].append(post)
+                self.site.posts_per_month[
+                    '{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post)
+                for tag in post.alltags:
+                    _tag_slugified = utils.slugify(tag)
+                    if _tag_slugified in slugged_tags:
+                        if tag not in self.site.posts_per_tag:
+                            # Tags that differ only in case
+                            other_tag = [existing for existing in self.site.posts_per_tag.keys() if utils.slugify(existing) == _tag_slugified][0]
+                            utils.LOGGER.error('You have tags that are too similar: {0} and {1}'.format(tag, other_tag))
+                            utils.LOGGER.error('Tag {0} is used in: {1}'.format(tag, post.source_path))
+                            utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join([p.source_path for p in self.site.posts_per_tag[other_tag]])))
+                            quit = True
+                    else:
+                        slugged_tags.add(utils.slugify(tag, force=True))
+                    self.site.posts_per_tag[tag].append(post)
+                self.site.posts_per_category[post.meta('category')].append(post)
+
+            if post.is_post:
+                # unpublished posts
+                self.site.all_posts.append(post)
+            else:
+                self.site.pages.append(post)
+
+            for lang in self.site.config['TRANSLATIONS'].keys():
+                self.site.post_per_file[post.destination_path(lang=lang)] = post
+                self.site.post_per_file[post.destination_path(lang=lang, extension=post.source_ext())] = post
+
+        # Sort everything.
+        self.timeline.sort(key=lambda p: p.date)
+        self.timeline.reverse()
+        self.posts.sort(key=lambda p: p.date)
+        self.posts.reverse()
+        self.all_posts.sort(key=lambda p: p.date)
+        self.all_posts.reverse()
+        self.pages.sort(key=lambda p: p.date)
+        self.pages.reverse()
+
+        for i, p in enumerate(self.site.posts[1:]):
+            p.next_post = self.site.posts[i]
+        for i, p in enumerate(self.site.posts[:-1]):
+            p.prev_post = self.site.posts[i + 1]
+        self.site._scanned = True
+        if not self.site.quiet:
+            print("done!", file=sys.stderr)
+
+        signal('scanned').send(self)
+
+        if quit and not ignore_quit:
+            sys.exit(1)
 
     def generic_page_renderer(self, lang, post, filters):
         """Render post fragments to final HTML pages."""
diff --git a/nikola/plugins/task/scan_posts.py b/nikola/plugins/task/scan_posts.py
index 0c9dc81e59..a0a4a6d6c7 100644
--- a/nikola/plugins/task/scan_posts.py
+++ b/nikola/plugins/task/scan_posts.py
@@ -51,22 +51,14 @@ def scan(self):
             "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'],
             "demote_headers": self.site.config['DEMOTE_HEADERS'],
         }
-        self.site.global_data = {}
-        self.site.posts = []
-        self.site.all_posts = []
-        self.site.posts_per_year = defaultdict(list)
-        self.site.posts_per_month = defaultdict(list)
-        self.site.posts_per_tag = defaultdict(list)
-        self.site.posts_per_category = defaultdict(list)
-        self.site.post_per_file = {}
-        self.site.timeline = []
-        self.site.pages = []
 
         seen = set([])
         if not self.site.quiet:
             print("Scanning posts", end='', file=sys.stderr)
 
-        slugged_tags = set([])
+        timeline = []
+        global_data = {}
+
         quit = False
         for wildcard, destination, template_name, use_in_feeds in \
                 self.site.config['post_pages']:
@@ -115,60 +107,7 @@ def scan(self):
                         template_name,
                         self.site.get_compiler(base_path)
                     )
-                    self.site.timeline.append(post)
-                    self.site.global_data[post.source_path] = post
-                    if post.use_in_feeds:
-                        self.site.posts.append(post)
-                        self.site.posts_per_year[
-                            str(post.date.year)].append(post)
-                        self.site.posts_per_month[
-                            '{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post)
-                        for tag in post.alltags:
-                            _tag_slugified = utils.slugify(tag)
-                            if _tag_slugified in slugged_tags:
-                                if tag not in self.site.posts_per_tag:
-                                    # Tags that differ only in case
-                                    other_tag = [existing for existing in self.site.posts_per_tag.keys() if utils.slugify(existing) == _tag_slugified][0]
-                                    utils.LOGGER.error('You have tags that are too similar: {0} and {1}'.format(tag, other_tag))
-                                    utils.LOGGER.error('Tag {0} is used in: {1}'.format(tag, post.source_path))
-                                    utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join([p.source_path for p in self.site.posts_per_tag[other_tag]])))
-                                    quit = True
-                            else:
-                                slugged_tags.add(utils.slugify(tag, force=True))
-                            self.site.posts_per_tag[tag].append(post)
-                        self.site.posts_per_category[post.meta('category')].append(post)
-
-                    if post.is_post:
-                        # unpublished posts
-                        self.site.all_posts.append(post)
-                    else:
-                        self.site.pages.append(post)
-
-                    for lang in self.site.config['TRANSLATIONS'].keys():
-                        self.site.post_per_file[post.destination_path(lang=lang)] = post
-                        self.site.post_per_file[post.destination_path(lang=lang, extension=post.source_ext())] = post
-
-        # Sort everything.
-        self.site.timeline.sort(key=lambda p: p.date)
-        self.site.timeline.reverse()
-        self.site.posts.sort(key=lambda p: p.date)
-        self.site.posts.reverse()
-        self.site.all_posts.sort(key=lambda p: p.date)
-        self.site.all_posts.reverse()
-        self.site.pages.sort(key=lambda p: p.date)
-        self.site.pages.reverse()
-
-        for i, p in enumerate(self.site.posts[1:]):
-            p.next_post = self.site.posts[i]
-        for i, p in enumerate(self.site.posts[:-1]):
-            p.prev_post = self.site.posts[i + 1]
-        self.site._scanned = True
-        if not self.site.quiet:
-            print("done!", file=sys.stderr)
-
-        signal('scanned').send(self)
-
-        if quit and not ignore_quit:
-            sys.exit(1)
+                    timeline.append(post)
+                    global_data[post.source_path] = post
 
-        yield self.group_task()
+        return timeline, global_data

From bf8f58f3999eb0a146d536e801ba2a94c6f109cc Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Sat, 9 May 2015 16:36:39 -0300
Subject: [PATCH 06/21] almost there

---
 nikola/nikola.py                  | 58 ++++++++++++++-----------------
 nikola/plugins/task/scan_posts.py |  1 -
 2 files changed, 26 insertions(+), 33 deletions(-)

diff --git a/nikola/nikola.py b/nikola/nikola.py
index 72c2a011b4..f443c65e4c 100644
--- a/nikola/nikola.py
+++ b/nikola/nikola.py
@@ -698,6 +698,7 @@ def __init__(self, **config):
             plugin_info.plugin_object.short_help = plugin_info.description
             self._commands[plugin_info.name] = plugin_info.plugin_object
 
+        self._activate_plugins_of_category("PostScanner")
         self._activate_plugins_of_category("Task")
         self._activate_plugins_of_category("LateTask")
         self._activate_plugins_of_category("TaskMultiplier")
@@ -1376,62 +1377,55 @@ def scan_posts(self):
             self.timeline.extend(timeline)
             self.global_data.update(global_data)
 
+        # Classify posts per year/tag/month/whatever
         slugged_tags = set([])
         for post in self.timeline:
             if post.use_in_feeds:
-                self.site.posts.append(post)
-                self.site.posts_per_year[
-                    str(post.date.year)].append(post)
-                self.site.posts_per_month[
+                self.posts.append(post)
+                self.posts_per_year[str(post.date.year)].append(post)
+                self.posts_per_month[
                     '{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post)
                 for tag in post.alltags:
                     _tag_slugified = utils.slugify(tag)
                     if _tag_slugified in slugged_tags:
-                        if tag not in self.site.posts_per_tag:
+                        if tag not in self.posts_per_tag:
                             # Tags that differ only in case
-                            other_tag = [existing for existing in self.site.posts_per_tag.keys() if utils.slugify(existing) == _tag_slugified][0]
+                            other_tag = [existing for existing in self.posts_per_tag.keys() if utils.slugify(existing) == _tag_slugified][0]
                             utils.LOGGER.error('You have tags that are too similar: {0} and {1}'.format(tag, other_tag))
                             utils.LOGGER.error('Tag {0} is used in: {1}'.format(tag, post.source_path))
-                            utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join([p.source_path for p in self.site.posts_per_tag[other_tag]])))
+                            utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join([p.source_path for p in self.posts_per_tag[other_tag]])))
                             quit = True
                     else:
                         slugged_tags.add(utils.slugify(tag, force=True))
-                    self.site.posts_per_tag[tag].append(post)
-                self.site.posts_per_category[post.meta('category')].append(post)
+                    self.posts_per_tag[tag].append(post)
+                self.posts_per_category[post.meta('category')].append(post)
 
             if post.is_post:
                 # unpublished posts
-                self.site.all_posts.append(post)
+                self.all_posts.append(post)
             else:
-                self.site.pages.append(post)
+                self.pages.append(post)
 
-            for lang in self.site.config['TRANSLATIONS'].keys():
-                self.site.post_per_file[post.destination_path(lang=lang)] = post
-                self.site.post_per_file[post.destination_path(lang=lang, extension=post.source_ext())] = post
+            for lang in self.config['TRANSLATIONS'].keys():
+                self.post_per_file[post.destination_path(lang=lang)] = post
+                self.post_per_file[post.destination_path(lang=lang, extension=post.source_ext())] = post
 
         # Sort everything.
-        self.timeline.sort(key=lambda p: p.date)
-        self.timeline.reverse()
-        self.posts.sort(key=lambda p: p.date)
-        self.posts.reverse()
-        self.all_posts.sort(key=lambda p: p.date)
-        self.all_posts.reverse()
-        self.pages.sort(key=lambda p: p.date)
-        self.pages.reverse()
-
-        for i, p in enumerate(self.site.posts[1:]):
-            p.next_post = self.site.posts[i]
-        for i, p in enumerate(self.site.posts[:-1]):
-            p.prev_post = self.site.posts[i + 1]
-        self.site._scanned = True
-        if not self.site.quiet:
+
+        for thing in self.timeline, self.posts, self.all_posts, self.pages:
+            thing.sort(key=lambda p: p.date)
+            thing.reverse()
+
+        for i, p in enumerate(self.posts[1:]):
+            p.next_post = self.posts[i]
+        for i, p in enumerate(self.posts[:-1]):
+            p.prev_post = self.posts[i + 1]
+        self._scanned = True
+        if not self.quiet:
             print("done!", file=sys.stderr)
 
         signal('scanned').send(self)
 
-        if quit and not ignore_quit:
-            sys.exit(1)
-
     def generic_page_renderer(self, lang, post, filters):
         """Render post fragments to final HTML pages."""
         context = {}
diff --git a/nikola/plugins/task/scan_posts.py b/nikola/plugins/task/scan_posts.py
index a0a4a6d6c7..2875cfcb39 100644
--- a/nikola/plugins/task/scan_posts.py
+++ b/nikola/plugins/task/scan_posts.py
@@ -59,7 +59,6 @@ def scan(self):
         timeline = []
         global_data = {}
 
-        quit = False
         for wildcard, destination, template_name, use_in_feeds in \
                 self.site.config['post_pages']:
             if not self.site.quiet:

From 70103adfdab8c0138f9d5e75840c3717b83e5b97 Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Sat, 9 May 2015 16:39:00 -0300
Subject: [PATCH 07/21] keep compatibility

---
 nikola/nikola.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/nikola/nikola.py b/nikola/nikola.py
index f443c65e4c..bd26305ad8 100644
--- a/nikola/nikola.py
+++ b/nikola/nikola.py
@@ -1353,10 +1353,13 @@ def flatten(task):
             'task_dep': task_dep
         }
 
-    def scan_posts(self):
-        """Scan all the posts."""
+    def scan_posts(self, really=False, ignore_quit=False, quiet=False):
+        """Scan all the posts.
+
+        Ignoring ignore_quit and quiet right now because noone uses them?
+        """
         # FIXME this is temporary while moving things out to a plugin
-        if self._scanned:
+        if self._scanned and not really:
             return
 
         # Reset things

From bc0e8152de70b43bddcb72581cb1152bfc1d8d7b Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Sat, 9 May 2015 16:59:38 -0300
Subject: [PATCH 08/21] flake8, support quit/ignore_quit

---
 nikola/nikola.py                  |  6 ++++--
 nikola/plugins/task/scan_posts.py | 12 +-----------
 2 files changed, 5 insertions(+), 13 deletions(-)

diff --git a/nikola/nikola.py b/nikola/nikola.py
index bd26305ad8..6609ce79fe 100644
--- a/nikola/nikola.py
+++ b/nikola/nikola.py
@@ -1356,7 +1356,7 @@ def flatten(task):
     def scan_posts(self, really=False, ignore_quit=False, quiet=False):
         """Scan all the posts.
 
-        Ignoring ignore_quit and quiet right now because noone uses them?
+        Ignoring quiet.
         """
         # FIXME this is temporary while moving things out to a plugin
         if self._scanned and not really:
@@ -1380,6 +1380,7 @@ def scan_posts(self, really=False, ignore_quit=False, quiet=False):
             self.timeline.extend(timeline)
             self.global_data.update(global_data)
 
+        quit = False
         # Classify posts per year/tag/month/whatever
         slugged_tags = set([])
         for post in self.timeline:
@@ -1426,7 +1427,8 @@ def scan_posts(self, really=False, ignore_quit=False, quiet=False):
         self._scanned = True
         if not self.quiet:
             print("done!", file=sys.stderr)
-
+        if quit and not ignore_quit:
+            sys.exit(1)
         signal('scanned').send(self)
 
     def generic_page_renderer(self, lang, post, filters):
diff --git a/nikola/plugins/task/scan_posts.py b/nikola/plugins/task/scan_posts.py
index 2875cfcb39..77d34faae5 100644
--- a/nikola/plugins/task/scan_posts.py
+++ b/nikola/plugins/task/scan_posts.py
@@ -25,13 +25,10 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 from __future__ import unicode_literals, print_function
-from collections import defaultdict
 import glob
 import os
 import sys
 
-from blinker import signal
-
 from nikola.plugin_categories import PostScanner
 from nikola import utils
 from nikola.post import Post
@@ -44,13 +41,6 @@ class ScanPosts(PostScanner):
 
     def scan(self):
         """Build final pages from metadata and HTML fragments."""
-        kw = {
-            "post_pages": self.site.config["post_pages"],
-            "translations": self.site.config["TRANSLATIONS"],
-            "filters": self.site.config["FILTERS"],
-            "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'],
-            "demote_headers": self.site.config['DEMOTE_HEADERS'],
-        }
 
         seen = set([])
         if not self.site.quiet:
@@ -64,7 +54,7 @@ def scan(self):
             if not self.site.quiet:
                 print(".", end='', file=sys.stderr)
             dirname = os.path.dirname(wildcard)
-            for dirpath, _, _ in os.walk(dirname, followlinks=True ):
+            for dirpath, _, _ in os.walk(dirname, followlinks=True):
                 dest_dir = os.path.normpath(os.path.join(destination,
                                             os.path.relpath(dirpath, dirname)))  # output/destination/foo/
                 # Get all the untranslated paths

From 2fe0484cdb3885e4cbd4cc75f64af6ee53a5c595 Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Sat, 9 May 2015 17:02:03 -0300
Subject: [PATCH 09/21] updated plugin doc

---
 nikola/plugin_categories.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/nikola/plugin_categories.py b/nikola/plugin_categories.py
index 51265d2022..2fa7d4d719 100644
--- a/nikola/plugin_categories.py
+++ b/nikola/plugin_categories.py
@@ -86,7 +86,11 @@ class PostScanner(BasePlugin):
     """The scan method of these plugins is called by Nikola.scan_posts."""
 
     def scan(self):
-        """Load posts into the timeline."""
+        """Creates a list of posts from some source. Returns timeline, global_data:
+
+        timeline is a list of Post objects
+        global_data is a dictionary where keys are source paths and values are posts. 
+        """
         raise NotImplementedError()
 
 

From b8cec997d192f6785b28512b3eb3d118e513c9f2 Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Sat, 9 May 2015 17:03:35 -0300
Subject: [PATCH 10/21] lie a little less

---
 docs/internals.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/internals.txt b/docs/internals.txt
index 9de56dff2c..0a19bb5374 100644
--- a/docs/internals.txt
+++ b/docs/internals.txt
@@ -95,8 +95,8 @@ posts are added into RSS feeds and stories are not. All of them are in a list ca
 "the timeline" formed by objects of class ``Post``.
 
 When you are creating a task that needs the list of posts and/or stories (for example,
-the RSS creation plugin) on task execution time, your plugin should have a dependency 
-on the ``scan_posts`` task to ensure the timeline is created and available in 
+the RSS creation plugin) on task execution time, your plugin should call ``self.site.scan_posts()``
+in ``gen_tasks`` to ensure the timeline is created and available in 
 ``self.site.timeline``. You should not modify the timeline, because it will cause consistency issues.
 
 .. sidebar:: scan_posts

From 0bd330d2108e19f0fd80bb6bff32046e89422750 Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Sat, 9 May 2015 17:17:53 -0300
Subject: [PATCH 11/21] removed a fixme

---
 nikola/nikola.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/nikola/nikola.py b/nikola/nikola.py
index 6609ce79fe..19236544ef 100644
--- a/nikola/nikola.py
+++ b/nikola/nikola.py
@@ -1358,7 +1358,6 @@ def scan_posts(self, really=False, ignore_quit=False, quiet=False):
 
         Ignoring quiet.
         """
-        # FIXME this is temporary while moving things out to a plugin
         if self._scanned and not really:
             return
 

From 086bc24b8735044b7c9fb92d0354ae0721aac4bf Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Sat, 9 May 2015 17:33:02 -0300
Subject: [PATCH 12/21] trailing whitespace

---
 nikola/nikola.py                  | 5 +----
 nikola/plugin_categories.py       | 2 +-
 nikola/plugins/task/scan_posts.py | 4 +---
 3 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/nikola/nikola.py b/nikola/nikola.py
index 19236544ef..ad08807d2d 100644
--- a/nikola/nikola.py
+++ b/nikola/nikola.py
@@ -266,7 +266,6 @@ def __init__(self, **config):
         }
 
         self.strict = False
-        self.global_data = {}
         self.posts = []
         self.all_posts = []
         self.posts_per_year = defaultdict(list)
@@ -1362,7 +1361,6 @@ def scan_posts(self, really=False, ignore_quit=False, quiet=False):
             return
 
         # Reset things
-        self.global_data = {}
         self.posts = []
         self.all_posts = []
         self.posts_per_year = defaultdict(list)
@@ -1374,10 +1372,9 @@ def scan_posts(self, really=False, ignore_quit=False, quiet=False):
         self.pages = []
 
         for p in self.plugin_manager.getPluginsOfCategory('PostScanner'):
-            timeline, global_data = p.plugin_object.scan()
+            timeline = p.plugin_object.scan()
             # FIXME: can there be conflicts here?
             self.timeline.extend(timeline)
-            self.global_data.update(global_data)
 
         quit = False
         # Classify posts per year/tag/month/whatever
diff --git a/nikola/plugin_categories.py b/nikola/plugin_categories.py
index 2fa7d4d719..e4eec6f1ee 100644
--- a/nikola/plugin_categories.py
+++ b/nikola/plugin_categories.py
@@ -89,7 +89,7 @@ def scan(self):
         """Creates a list of posts from some source. Returns timeline, global_data:
 
         timeline is a list of Post objects
-        global_data is a dictionary where keys are source paths and values are posts. 
+        global_data is a dictionary where keys are source paths and values are posts.
         """
         raise NotImplementedError()
 
diff --git a/nikola/plugins/task/scan_posts.py b/nikola/plugins/task/scan_posts.py
index 77d34faae5..308bb47ec5 100644
--- a/nikola/plugins/task/scan_posts.py
+++ b/nikola/plugins/task/scan_posts.py
@@ -47,7 +47,6 @@ def scan(self):
             print("Scanning posts", end='', file=sys.stderr)
 
         timeline = []
-        global_data = {}
 
         for wildcard, destination, template_name, use_in_feeds in \
                 self.site.config['post_pages']:
@@ -97,6 +96,5 @@ def scan(self):
                         self.site.get_compiler(base_path)
                     )
                     timeline.append(post)
-                    global_data[post.source_path] = post
 
-        return timeline, global_data
+        return timeline

From a55a074fdb4c225ceb2a9f233c1122a48b04f1cc Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Sat, 9 May 2015 17:45:49 -0300
Subject: [PATCH 13/21] fix tests

---
 nikola/nikola.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nikola/nikola.py b/nikola/nikola.py
index ad08807d2d..ca53770b10 100644
--- a/nikola/nikola.py
+++ b/nikola/nikola.py
@@ -53,6 +53,7 @@
 from yapsy.PluginManager import PluginManager
 from blinker import signal
 
+from .post import Post
 from . import DEBUG, utils
 from .plugin_categories import (
     Command,

From 7cc53c2068a60087ef4de5ea5b176b902aa08658 Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Sat, 9 May 2015 17:57:14 -0300
Subject: [PATCH 14/21] SHUTUP FLAKE8

---
 nikola/nikola.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nikola/nikola.py b/nikola/nikola.py
index ca53770b10..b12276d89a 100644
--- a/nikola/nikola.py
+++ b/nikola/nikola.py
@@ -53,7 +53,7 @@
 from yapsy.PluginManager import PluginManager
 from blinker import signal
 
-from .post import Post
+from .post import Post  # NOQA
 from . import DEBUG, utils
 from .plugin_categories import (
     Command,

From c809cdef59e4e89928c7ebdb3f274807a0a8d6fa Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Sat, 9 May 2015 21:37:15 -0300
Subject: [PATCH 15/21] better docstrings

---
 nikola/plugin_categories.py       | 6 +-----
 nikola/plugins/task/scan_posts.py | 2 +-
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/nikola/plugin_categories.py b/nikola/plugin_categories.py
index e4eec6f1ee..bd47551c5c 100644
--- a/nikola/plugin_categories.py
+++ b/nikola/plugin_categories.py
@@ -86,11 +86,7 @@ class PostScanner(BasePlugin):
     """The scan method of these plugins is called by Nikola.scan_posts."""
 
     def scan(self):
-        """Creates a list of posts from some source. Returns timeline, global_data:
-
-        timeline is a list of Post objects
-        global_data is a dictionary where keys are source paths and values are posts.
-        """
+        """Creates a list of posts from some source. Returns a list of Post objects."""
         raise NotImplementedError()
 
 
diff --git a/nikola/plugins/task/scan_posts.py b/nikola/plugins/task/scan_posts.py
index 308bb47ec5..a6f04e6156 100644
--- a/nikola/plugins/task/scan_posts.py
+++ b/nikola/plugins/task/scan_posts.py
@@ -40,7 +40,7 @@ class ScanPosts(PostScanner):
     name = "scan_posts"
 
     def scan(self):
-        """Build final pages from metadata and HTML fragments."""
+        """Create list of posts from POSTS and PAGES options."""
 
         seen = set([])
         if not self.site.quiet:

From 5060eacf40587d93739c3fd55f3957e7401befd0 Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Sun, 10 May 2015 11:09:28 -0300
Subject: [PATCH 16/21] not a task

---
 nikola/plugins/{task => misc}/scan_posts.plugin | 0
 nikola/plugins/{task => misc}/scan_posts.py     | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename nikola/plugins/{task => misc}/scan_posts.plugin (100%)
 rename nikola/plugins/{task => misc}/scan_posts.py (100%)

diff --git a/nikola/plugins/task/scan_posts.plugin b/nikola/plugins/misc/scan_posts.plugin
similarity index 100%
rename from nikola/plugins/task/scan_posts.plugin
rename to nikola/plugins/misc/scan_posts.plugin
diff --git a/nikola/plugins/task/scan_posts.py b/nikola/plugins/misc/scan_posts.py
similarity index 100%
rename from nikola/plugins/task/scan_posts.py
rename to nikola/plugins/misc/scan_posts.py

From 422e0e1abfba2ae56330c08e992cea33931d6946 Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Sun, 10 May 2015 11:12:44 -0300
Subject: [PATCH 17/21] updated changelog

---
 CHANGES.txt | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 752e18d00e..84217875e4 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,12 @@
+New in master
+=============
+
+Bugfixes
+--------
+
+* Scanning of posts refactored out of core (Issue #1700)
+
+
 New in v7.4.1
 =============
 
@@ -11,7 +20,6 @@ Features
 Bugfixes
 --------
 
-* Scanning of posts refactored out of core (Issue #1700)
 * Handle strange URLs, like ed2k:// (Issue #1695)
 * Fix very old metadata format support (Issue #1689)
 

From 3b08b41e1f12034b9883258b671067eaccbcc666 Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Tue, 19 May 2015 17:13:03 -0300
Subject: [PATCH 18/21] merged master

---
 nikola/nikola.py | 87 ------------------------------------------------
 1 file changed, 87 deletions(-)

diff --git a/nikola/nikola.py b/nikola/nikola.py
index ee140a4c1b..519643834b 100644
--- a/nikola/nikola.py
+++ b/nikola/nikola.py
@@ -1432,7 +1432,6 @@ def scan_posts(self, really=False, ignore_quit=False, quiet=False):
             self.timeline.extend(timeline)
 
         quit = False
-<<<<<<< HEAD
         # Classify posts per year/tag/month/whatever
         slugged_tags = set([])
         for post in self.timeline:
@@ -1451,81 +1450,6 @@ def scan_posts(self, really=False, ignore_quit=False, quiet=False):
                             utils.LOGGER.error('Tag {0} is used in: {1}'.format(tag, post.source_path))
                             utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join([p.source_path for p in self.posts_per_tag[other_tag]])))
                             quit = True
-=======
-        for wildcard, destination, template_name, use_in_feeds in \
-                self.config['post_pages']:
-            if not self.quiet and not quiet:
-                print(".", end='', file=sys.stderr)
-            dirname = os.path.dirname(wildcard)
-            for dirpath, _, _ in os.walk(dirname, followlinks=True):
-                dest_dir = os.path.normpath(os.path.join(destination,
-                                            os.path.relpath(dirpath, dirname)))  # output/destination/foo/
-                # Get all the untranslated paths
-                dir_glob = os.path.join(dirpath, os.path.basename(wildcard))  # posts/foo/*.rst
-                untranslated = glob.glob(dir_glob)
-                # And now get all the translated paths
-                translated = set([])
-                for lang in self.config['TRANSLATIONS'].keys():
-                    if lang == self.config['DEFAULT_LANG']:
-                        continue
-                    lang_glob = utils.get_translation_candidate(self.config, dir_glob, lang)  # posts/foo/*.LANG.rst
-                    translated = translated.union(set(glob.glob(lang_glob)))
-                # untranslated globs like *.rst often match translated paths too, so remove them
-                # and ensure x.rst is not in the translated set
-                untranslated = set(untranslated) - translated
-
-                # also remove from translated paths that are translations of
-                # paths in untranslated_list, so x.es.rst is not in the untranslated set
-                for p in untranslated:
-                    translated = translated - set([utils.get_translation_candidate(self.config, p, l) for l in self.config['TRANSLATIONS'].keys()])
-
-                full_list = list(translated) + list(untranslated)
-                # We eliminate from the list the files inside any .ipynb folder
-                full_list = [p for p in full_list
-                             if not any([x.startswith('.')
-                                         for x in p.split(os.sep)])]
-
-                for base_path in full_list:
-                    if base_path in seen:
-                        continue
-                    else:
-                        seen.add(base_path)
-                    post = Post(
-                        base_path,
-                        self.config,
-                        dest_dir,
-                        use_in_feeds,
-                        self.MESSAGES,
-                        template_name,
-                        self.get_compiler(base_path)
-                    )
-                    self.timeline.append(post)
-                    self.global_data[post.source_path] = post
-                    if post.use_in_feeds:
-                        self.posts.append(post)
-                        self.posts_per_year[
-                            str(post.date.year)].append(post)
-                        self.posts_per_month[
-                            '{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post)
-                        for tag in post.alltags:
-                            _tag_slugified = utils.slugify(tag)
-                            if _tag_slugified in slugged_tags:
-                                if tag not in self.posts_per_tag:
-                                    # Tags that differ only in case
-                                    other_tag = [existing for existing in self.posts_per_tag.keys() if utils.slugify(existing) == _tag_slugified][0]
-                                    utils.LOGGER.error('You have tags that are too similar: {0} and {1}'.format(tag, other_tag))
-                                    utils.LOGGER.error('Tag {0} is used in: {1}'.format(tag, post.source_path))
-                                    utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join([p.source_path for p in self.posts_per_tag[other_tag]])))
-                                    quit = True
-                            else:
-                                slugged_tags.add(utils.slugify(tag, force=True))
-                            self.posts_per_tag[tag].append(post)
-                        self._add_post_to_category(post, post.meta('category'))
-
-                    if post.is_post:
-                        # unpublished posts
-                        self.all_posts.append(post)
->>>>>>> master
                     else:
                         slugged_tags.add(utils.slugify(tag, force=True))
                     self.posts_per_tag[tag].append(post)
@@ -1542,22 +1466,11 @@ def scan_posts(self, really=False, ignore_quit=False, quiet=False):
                 self.post_per_file[post.destination_path(lang=lang, extension=post.source_ext())] = post
 
         # Sort everything.
-<<<<<<< HEAD
 
         for thing in self.timeline, self.posts, self.all_posts, self.pages:
             thing.sort(key=lambda p: p.date)
             thing.reverse()
-=======
-        self.timeline.sort(key=lambda p: p.date)
-        self.timeline.reverse()
-        self.posts.sort(key=lambda p: p.date)
-        self.posts.reverse()
-        self.all_posts.sort(key=lambda p: p.date)
-        self.all_posts.reverse()
-        self.pages.sort(key=lambda p: p.date)
-        self.pages.reverse()
         self._sort_category_hierarchy()
->>>>>>> master
 
         for i, p in enumerate(self.posts[1:]):
             p.next_post = self.posts[i]

From d5c284bbf63b9b116a676e6bb05afdefa9e612ff Mon Sep 17 00:00:00 2001
From: Felix Fontein <felix@fontein.de>
Date: Wed, 20 May 2015 08:18:38 +0200
Subject: [PATCH 19/21] Replicated change from commit 64e6ed8.

---
 nikola/nikola.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nikola/nikola.py b/nikola/nikola.py
index 519643834b..59f9ebf252 100644
--- a/nikola/nikola.py
+++ b/nikola/nikola.py
@@ -1453,7 +1453,7 @@ def scan_posts(self, really=False, ignore_quit=False, quiet=False):
                     else:
                         slugged_tags.add(utils.slugify(tag, force=True))
                     self.posts_per_tag[tag].append(post)
-                self.posts_per_category[post.meta('category')].append(post)
+                self._add_post_to_category(post, post.meta('category'))
 
             if post.is_post:
                 # unpublished posts

From 2f9436d2c96923dc4d70422ea5b6f407ed7bfd9e Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Wed, 20 May 2015 09:11:56 -0300
Subject: [PATCH 20/21] describe PostScanner plugins

---
 docs/extending.txt | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/extending.txt b/docs/extending.txt
index 12e69fa284..0c5721d0a4 100644
--- a/docs/extending.txt
+++ b/docs/extending.txt
@@ -432,6 +432,13 @@ Does nothing specific, can be used to modify the site object (and thus the confi
 Put all the magic you want in ``set_site()``, and don’t forget to run the one
 from ``super()``.
 
+PostScanner Plugins
+-------------------
+
+Get posts and stories from "somewhere" to be added to the timeline.
+The only currently existing plugin of this kind reads them from disk.
+
+
 Plugin Index
 ============
 

From c8ff4153992d7fcf360b7ad3ec5f52424d2dca73 Mon Sep 17 00:00:00 2001
From: Roberto Alsina <ralsina@netmanagers.com.ar>
Date: Fri, 22 May 2015 13:37:22 -0300
Subject: [PATCH 21/21] broken symlink

---
 docs/sphinx/upgrading-to-v6.txt | 1 -
 1 file changed, 1 deletion(-)
 delete mode 120000 docs/sphinx/upgrading-to-v6.txt

diff --git a/docs/sphinx/upgrading-to-v6.txt b/docs/sphinx/upgrading-to-v6.txt
deleted file mode 120000
index 1c7e16f96b..0000000000
--- a/docs/sphinx/upgrading-to-v6.txt
+++ /dev/null
@@ -1 +0,0 @@
-../upgrading-to-v6.txt
\ No newline at end of file