Merge 1bc2a31 into 8b6f349

nikola/nikola.py

@@ -73,6 +73,7 @@
     SignalHandler,
     ConfigPlugin,
     PostScanner,
+    Taxonomy,
 )
 
 if DEBUG:
@@ -948,6 +949,7 @@ def init_plugins(self, commands_only=False, load_all=False):
             "SignalHandler": SignalHandler,
             "ConfigPlugin": ConfigPlugin,
             "PostScanner": PostScanner,
+            "Taxonomy": Taxonomy,
         })
         self.plugin_manager.getPluginLocator().setPluginInfoExtension('plugin')
         extra_plugins_dirs = self.config['EXTRA_PLUGINS_DIRS']
@@ -1019,6 +1021,7 @@ def plugin_position_in_places(plugin):
 
         self.plugin_manager.loadPlugins()
 
+        self._activate_plugins_of_category("Taxonomy")
         self._activate_plugins_of_category("SignalHandler")
 
         # Emit signal for SignalHandlers which need to start running immediately.

nikola/plugin_categories.py

@@ -49,6 +49,7 @@
     'SignalHandler',
     'ConfigPlugin',
     'PostScanner',
+    'Taxonomy',
 )
 
 
@@ -451,3 +452,133 @@ def import_file(self):
     def save_post(self):
         """Save a post to disk."""
         raise NotImplementedError()
+
+
+class Taxonomy(BasePlugin):
+    """Taxonomy for posts.
+
+    A taxonomy plugin allows to classify posts (see #2107) by
+    classification strings.
+    """
+
+    name = "dummy_taxonomy"
+
+    # Adjust the following values in your plugin!
+
+    # The classification name to be used for path handlers.
+    classification_name = "taxonomy"
+    # The classification name to be used when storing the classification
+    # in the metadata.
+    metadata_name = "taxonomy"
+    # If True, there can be more than one classification per post; in that case,
+    # the classification data in the metadata is stored as a list. If False,
+    # the classification data in the metadata is stored as a string, or None
+    # when no classification is given.
+    more_than_one_classifications_per_post = False
+    # Whether the classification has a hierarchy.
+    has_hierarchy = False
+    # If True, the list for a classification includes all posts with a
+    # sub-classification (in case has_hierarchy is True).
+    include_posts_from_subhierarchies = False
+    # Whether to show the posts for one classification as an index or
+    # as a post list.
+    show_list_as_index = False
+    # The template to use for the post list for one classification.
+    # Set to none to avoid generating overviews.
+    template_for_list_of_one_classification = "tagindex.tmpl"
+    # The template to use for the classification overview page.
+    template_for_classification_overview = "list.tmpl"
+    # Whether this classification applies to posts.
+    apply_to_posts = True
+    # Whether this classification applies to pages.
+    apply_to_pages = False
+    # The minimum number of posts a classification must have to be listed in
+    # the overview.
+    minimum_post_count_per_classification_in_overview = 1
+    # Whether post lists resp. indexes should be created for empty
+    # classifications.
+    omit_empty_classifications = False
+    # Whether to include all classifications for all languages in every
+    # language, or only the classifications for one language in its language's
+    # pages.
+    also_create_classifications_from_other_languages = True
+
+    def classify(self, post, lang):
+        """Classify the given post for the given language.
+
+        Must return a list or tuple of strings.
+        """
+        raise NotImplementedError()
+
+    def sort_posts(self, posts, lang):
+        """Sort the given list of posts."""
+        pass
+
+    def get_list_path(self, lang):
+        """A path handler for the list of all classifications.
+
+        The last element in the returned path must have no extension, and the
+        PRETTY_URLS config must be ignored. The return value will be modified
+        based on the PRETTY_URLS and INDEX_FILE settings.
+        """
+        raise NotImplementedError()
+
+    def get_path(self, classification, lang):
+        """A path handler for the given classification.
+
+        The last element in the returned path must have no extension, and the
+        PRETTY_URLS config must be ignored. The return value will be modified
+        based on the PRETTY_URLS and INDEX_FILE settings.
+
+        For hierarchical taxonomies, the result of extract_hierarchy is provided.
+        For non-hierarchical taxonomies, the classification string itself is provided.
+        """
+        raise NotImplementedError()
+
+    def extract_hierarchy(self, classification):
+        """Given a classification, return a list of parts in the hierarchy.
+
+        For non-hierarchical taxonomies, it usually suffices to return
+        `[classification]`.
+        """
+        return [classification]
+
+    def recombine_classification_from_hierarchy(self, hierarchy):
+        """Given a list of parts in the hierarchy, return the classification string.
+
+        For non-hierarchical taxonomies, it usually suffices to return hierarchy[0].
+        """
+        return hierarchy[0]
+
+    def provide_list_context_and_uptodate(self):
+        """Provide data for the context and the uptodate list for the list of all classifiations.
+
+        Must return a tuple of two dicts. The first is merged into the page's context,
+        the second will be put into the uptodate list of all generated tasks.
+
+        Context must contain `title`.
+        """
+        raise NotImplementedError()
+
+    def provide_context_and_uptodate(self, classification):
+        """Provide data for the context and the uptodate list for the list of the given classifiation.
+
+        Must return a tuple of two dicts. The first is merged into the page's context,
+        the second will be put into the uptodate list of all generated tasks.
+
+        Context must contain `title`, which should be something like 'Posts about <classification>',
+        and `classification_title`, which should be related to the classification string.
+        """
+        raise NotImplementedError()
+
+    def postprocess_posts_per_classification(self, posts_per_classification_per_language, flat_hierarchy_per_lang=None, hierarchy_lookup_per_lang=None):
+        """Rearrange, modify or otherwise use the list of posts per classification and per language.
+
+        For compatibility reasons, the list could be stored somewhere else as well.
+
+        In case `has_hierarchy` is `True`, `flat_hierarchy_per_lang` is the flat
+        hierarchy consisting of `TreeNode` elements, and `hierarchy_lookup_per_lang`
+        is the corresponding hierarchy lookup mapping classification strings to
+        `TreeNode` objects.
+        """
+        pass

nikola/plugins/misc/taxonomies_classifier.plugin

@@ -0,0 +1,12 @@
+[Core]
+name = classify_taxonomies
+module = classify_taxonomies
+
+[Documentation]
+author = Roberto Alsina
+version = 1.0
+website = https://getnikola.com/
+description = Classifies the timeline into taxonomies.
+
+[Nikola]
+plugincategory = SignalHandler

nikola/plugins/misc/taxonomies_classifier.py

@@ -0,0 +1,210 @@
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012-2016 Roberto Alsina and others.
+
+# Permission is hereby granted, free of charge, to any
+# person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the
+# Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the
+# Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice
+# shall be included in all copies or substantial portions of
+# the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""Render the taxonomy overviews, classification pages and feeds."""
+
+from __future__ import unicode_literals
+import blinker
+import natsort
+import os
+import sys
+
+from collections import defaultdict
+
+from nikola.plugin_categories import SignalHandler
+from nikola import utils
+
+
+class TaxonomiesClassifier(SignalHandler):
+    """Render the tag/category pages and feeds."""
+
+    name = "render_taxonomies"
+
+    def _do_classification(self):
+        taxonomies = self.site.plugin_manager.getPluginsOfCategory('Taxonomy')
+        self.site.posts_per_classification = {}
+        for taxonomy in taxonomies:
+            if taxonomy.classification_name in self.site.posts_per_classification:
+                raise Exception("Found more than one taxonomy with classification name '{}'!".format(taxonomy.classification_name))
+            self.site.posts_per_classification[taxonomy.classification_name] = {
+                lang: defaultdict(set) for lang in self.config['TRANSLATIONS'].keys()
+            }
+
+        # Classify posts
+        for post in self.timeline:
+            for taxonomy in taxonomies:
+                if taxonomy.apply_to_posts if post.is_post else taxonomy.apply_to_pages:
+                    classifications = {}
+                    for lang in self.config['TRANSLATIONS'].keys():
+                        # Extract classifications for this language
+                        classifications[lang] = taxonomy.classify(post, lang)
+                        assert taxonomy.more_than_one_classifications_per_post or len(classifications[lang]) <= 1
+                        # Store in metadata
+                        if taxonomy.more_than_one_classifications_per_post:
+                            post.meta[lang][taxonomy.metadata_name] = classifications[lang]
+                        else:
+                            post.meta[lang][taxonomy.metadata_name] = classifications[lang][0] if len(classifications[lang]) > 0 else None
+                        # Add post to sets
+                        for classification in classifications[lang]:
+                            while classification:
+                                self.site.posts_per_classification[taxonomy.classification_name][lang][classification].add(post)
+                                if not taxonomy.include_posts_from_subhierarchies or not taxonomy.has_hierarchy:
+                                    break
+                                classification = taxonomy.recombine_classification_from_hierarchy(taxonomy.extract_hierarchy(classification)[:-1])
+
+        # Check for valid paths and for collisions
+        taxonomy_outputs = {lang: dict() for lang in self.config['TRANSLATIONS'].keys()}
+        quit = False
+        for taxonomy in taxonomies:
+            # Check for collisions (per language)
+            for lang in self.config['TRANSLATIONS'].keys():
+                for tlang in self.config['TRANSLATIONS'].keys():
+                    if lang != tlang and not taxonomy.also_create_classifications_from_other_languages:
+                        continue
+                    for classification, posts in self.site.posts_per_classification[taxonomy.classification_name][tlang].items():
+                        # Obtain path as tuple
+                        if taxonomy.has_hierarchy:
+                            path = taxonomy.get_path(taxonomy.extract_hierarchy(classification), lang)
+                        else:
+                            path = taxonomy.get_path(classification, lang)
+                        path = tuple(path)
+                        # Check that path is OK
+                        for path_element in path:
+                            if len(path_element) == 0:
+                                utils.LOGGER.error("{0} {1} yields invalid path '{2}'!".format(taxonomy.classification_name.title(), classification, '/'.join(path)))
+                                quit = True
+                        # Determine collisions
+                        if path in taxonomy_outputs[lang]:
+                            other_classification_name, other_classification, other_posts = taxonomy_outputs[lang][path]
+                            utils.LOGGER.error('You have classifications that are too similar: {0} "{1}" and {1} "{2}"'.format(
+                                taxonomy.classification_name, classification, other_classification_name, other_classification))
+                            utils.LOGGER.error('{0} {1} is used in: {1}'.format(
+                                taxonomy.classification_name.title(), classification, ', '.join(sorted([p.source_path for p in posts]))))
+                            utils.LOGGER.error('{0} {1} is used in: {1}'.format(
+                                other_classification_name.title(), other_classification, ', '.join(sorted([p.source_path for p in other_posts]))))
+                            quit = True
+                        else:
+                            taxonomy_outputs[lang][path] = (taxonomy.classification_name, classification, posts)
+        if quit:
+            sys.exit(1)
+
+        # Sort everything.
+        self.site.flat_hierarchy_per_classification = {}
+        self.site.hierarchy_lookup_per_classification = {}
+        for taxonomy in taxonomies:
+            # Sort post lists
+            for lang, posts_per_classification in self.site.posts_per_classification[taxonomy.classification_name].items():
+                # Convert sets to lists and sort them
+                for classification in list(posts_per_classification.keys()):
+                    posts = list(posts_per_classification[classification])
+                    posts.sort(key=lambda p:
+                               (int(p.meta('priority')) if p.meta('priority') else 0,
+                                p.date, p.source_path))
+                    posts.reverse()
+                    posts_per_classification[classification] = posts
+            # Create hierarchy information
+            if taxonomy.has_hierarchy:
+                self.site.flat_hierarchy_per_classification[taxonomy.classification_name] = {}
+                self.site.hierarchy_lookup_per_classification[taxonomy.classification_name] = {}
+                for lang, posts_per_classification in self.site.posts_per_classification[taxonomy.classification_name].items():
+                    # Compose hierarchy
+                    hierarchy = {}
+                    for classification in posts_per_classification.keys():
+                        hier = taxonomy.extract_hierarchy(classification)
+                        node = hierarchy
+                        for he in hier:
+                            if he not in node:
+                                node[he] = {}
+                            node = node[he]
+                    hierarchy_lookup = {}
+
+                    def create_hierarchy(cat_hierarchy, parent=None):
+                        """Create category hierarchy."""
+                        result = []
+                        for name, children in cat_hierarchy.items():
+                            node = utils.TreeNode(name, parent)
+                            node.children = create_hierarchy(children, node)
+                            node.classification_path = [pn.name for pn in node.get_path()]
+                            node.classification_name = taxonomy.recombine_classification_from_hierarchy(node.classification_path)
+                            hierarchy_lookup[node.classification_name] = node
+                        return natsort.natsorted(result, key=lambda e: e.name, alg=natsort.ns.F | natsort.ns.IC)
+
+                    root_list = create_hierarchy(hierarchy)
+                    flat_hierarchy = utils.flatten_tree_structure(root_list)
+                    # Store result
+                    self.site.flat_hierarchy_per_classification[taxonomy.classification_name][lang] = flat_hierarchy
+                    self.site.hierarchy_lookup_per_classification[taxonomy.classification_name][lang] = hierarchy_lookup
+                taxonomy.postprocess_posts_per_classification(self.site.posts_per_classification[taxonomy.classification_name],
+                                                              self.site.flat_hierarchy_per_classification[taxonomy.classification_name],
+                                                              self.site.hierarchy_lookup_per_classification[taxonomy.classification_name])
+            else:
+                taxonomy.postprocess_posts_per_classification(self.site.posts_per_classification[taxonomy.classification_name], flat_hierarchy, hierarchy_lookup)
+
+    def _postprocess_path(self, path, lang, force_extension=None):
+        if force_extension is not None:
+            if len(path) == 0:
+                path = [os.path.splitext(self.site.config['INDEX_FILE'])[0]]
+            path[-1] += force_extension
+        elif self.site.config['PRETTY_URLS'] or len(path) == 0:
+            path = path + [self.site.config['INDEX_FILE']]
+        else:
+            path[-1] += '.html'
+        return [_f for _f in [self.site.config['TRANSLATIONS'][lang]] + path if _f]
+
+    def _taxonomy_index_path(self, lang, taxonomy):
+        """Return path to the classification overview."""
+        return self._postprocess_path(taxonomy.get_list_path(lang), lang)
+
+    def _taxonomy_path(self, name, lang, taxonomy, force_extension=None):
+        """Return path to a classification."""
+        if taxonomy.has_hirearchy:
+            path = taxonomy.get_path(taxonomy.extract_hierarchy(name), lang)
+        else:
+            path = taxonomy.get_path(name, lang)
+        return self._postprocess_path(path, lang, force_extension=force_extension)
+
+    def _taxonomy_atom_path(self, name, lang, taxonomy):
+        """Return path to a classification Atom feed."""
+        return self._taxonomy_path(name, lang, taxonomy, force_extension='.atom')
+
+    def _taxonomy_rss_path(self, name, lang, taxonomy):
+        """Return path to a classification RSS feed."""
+        return self._taxonomy_path(name, lang, taxonomy, force_extension='.xml')
+
+    def _register_path_handlers(self, taxonomy):
+        self.site.register_path_handler('{0}_index'.format(taxonomy.classification_name), lambda name, lang: self._tag_index_path(lang, taxonomy))
+        self.site.register_path_handler('{0}'.format(taxonomy.classification_name), lambda name, lang: self._tag_path(name, lang, taxonomy))
+        self.site.register_path_handler('{0}_atom'.format(taxonomy.classification_name), lambda name, lang: self._tag_atom_path(name, lang, taxonomy))
+        self.site.register_path_handler('{0}_rss'.format(taxonomy.classification_name), lambda name, lang: self._tag_rss_path(name, lang, taxonomy))
+
+    def set_site(self, site):
+        """Set site, which is a Nikola instance."""
+        super(TaxonomiesClassifier, self).set_site(site)
+        # Add hook for after post scanning
+        blinker.signal("scanned").connect(self._do_classification)
+        # Register path handlers
+        for taxonomy in self.plugin_manager.getPluginsOfCategory('Taxonomy'):
+            self._register_path_handlers(taxonomy)