Pivot resources using templates. (#1342)

* core: change pivoter to use templates instead of hard coded text extraction.
Issue #1076

* Remove test value for repivot.

* docs: add documentation for the pivot templates.

* docs: add references to pivot templates.

doc/cookbook/custom-pivot.rst

@@ -55,3 +55,6 @@ this::
 
 See :ref:`guide-datamodel-query-model` for more details on how to
 filter on custom pivot columns.
+
+See :ref:`cookbook-pivot-templates` to change the content of regular pivot columns and search texts.
+

doc/cookbook/index.rst

@@ -9,6 +9,7 @@ Cookbooks
    custom-action
    custom-model
    custom-pivot
+   pivot-templates
    custom-tag
    writing-module
    task-queue

doc/cookbook/pivot-templates.rst

@@ -0,0 +1,90 @@
+.. _cookbook-pivot-templates:
+
+Pivot Templates
+===============
+
+:ref:`guide-datamodel-query-model` uses database indices on special pivot
+columns and full text fields.
+
+These columns and text fields are extracted in a process called *pivoting*.
+This is done a short period after a resource has been updated.
+
+The content of the pivot columns is determined by a template. This template is
+called ``pivot/pivot.tpl`` (in ``mod_base``) and is rendered using a *catinclude*. 
+This makes it possible to have your own unique indexing per category.
+
+The ``pivot/pivot.tpl`` template consists of multiple blocks. Each block corresponds
+to a pivot column. The block is rendered for determining the pivot’s content.
+
+The following blocks are defined:
+
+**a, b, c and d**
+
+  These blocks are used for the full text index. Block *a* contains more
+  important texts than block *b*, *c* or *d*. Per default the title and other
+  essential texts are shown in block *a*, and block *d* is used for related
+  texts (think titles of connected resources).
+
+  The default pivot template uses the following catincluded templates for the
+  different blocks:
+
+   * ``pivot/_title_text.tpl`` for block *a*
+   * ``pivot/_main_text.tpl`` for block *b*
+   * ``pivot/_block_text.tpl`` for block *c*
+   * ``pivot/_related_text.tpl`` for block *c*
+
+**title**
+
+  The title used for database sorting etc. Per default the lowercased title in the selected
+  pivot language or the default site language. 
+
+**related_ids**
+
+  The ids of all related resources and categories. This is a full text index used for 
+  finding resources that are similar to some other resource.
+  All matching resource ids should be prefixed with ``zpo`` and category ids with ``zpc``
+  (for example ``zpo1234``).
+
+  The catincluded template ``pivot/_related_ids.tpl`` is used for extracting these ids.
+  This template also adds the id of the content group as a related resource.
+
+**address_street, address_city, address_postcode, address_state, address_country**
+
+  The address to be used for searches. Defaults to the related ``address_…`` properties
+  with a fallback to the ``mail_address_…`` properties. 
+
+  The country should be the two letter ISO code, not the country’s descriptive text.
+
+**name_first, name_surname**
+
+  If the resource is a person then this should contain the first name and surname.
+
+**gender**
+
+  The gender of the person, currently a single letter.  For example: ``f``, ``m`` or ``?``.
+
+**date_start, date_end**
+
+  The start or end date associated with the resource. This should be in a parseable format, which
+  will be done automatically when echoing a date directly. Make sure UTC is used as the timezone (note that the timezone will be set to UTC when pivoting).
+
+**date_start_month_day, date_end_month_day**
+
+  A four digit number representing the month and day for the date start and date end. Think of
+  an index for birthdays, without exposing the birth year.
+
+  Defaults to ``{{ id.date_start|date:"md" }}`` and ``{{ id.date_end|date:"md" }}``.
+
+**location_lat, location_lng**
+
+  The latitude and longitude of the resource location. This should be empty or a floating point number.
+
+  Defaults to ``{{ id.computed_location_lat|default:id.location_lat }}`` and ``{{ id.computed_location_lng|default:id.location_lng }}``
+
+**date_repivot**
+
+  The date and time this resource should be repivoted. This is useful if some resources (like events) should be indexed with lower priorities after a certain date. A pivot task will be scheduled at the specified date.
+  Per default no repivot will be scheduled.
+
+
+See :ref:`cookbook-custom-pivots` for adding your own custom pivot columns.

doc/developer-guide/resources.rst

@@ -125,13 +125,13 @@ Pivot columns
 If you want to search by or order on any custom defined
 property, you need to define your own database column in a *custom pivot*.
 
-.. seealso:: :ref:`cookbook-custom-pivots`
+.. seealso:: :ref:`cookbook-custom-pivots`, :ref:`cookbook-pivot-templates`
 
 .. note::
 
-    Zotonic is smart enough that when you enter any textual information
-    into any resource property, it will extract this and put it in the
-    ``pivot_tsv`` column, for use in full-text searches.
+    If you want to find resources on non standard content or texts, then 
+    you can change the texts that are pivoted. This can be done by adding
+    specific ``pivot.tpl`` templates. See :ref:`cookbook-pivot-templates`
 
 The pivot queue
 ^^^^^^^^^^^^^^^

doc/developer-guide/search.rst

@@ -83,7 +83,7 @@ Query-model arguments
 
   Filtering on columns.
 
-  ``filter=['pivot_title', 'Hello']``
+  ``filter=['pivot_title', 'hello']``
 
   In its most simple form, this does an 'equals' compare filter. The
   "filter" keywords expects a list. If the list is two elements long,
@@ -231,6 +231,8 @@ Query-model arguments
 
   ``sort='-rsc.modified'``
 
+.. seealso:: :ref:`cookbook-pivot-templates`
+
 **custompivot**
 
   Add a join on the given custom pivot table. The table is joined to
@@ -245,6 +247,7 @@ Query-model arguments
 
   ``{query custompivot="pivotname" filter=["pivot1.fieldname", `=`, "hello"]}``
 
+.. seealso:: :ref:`cookbook-custom-pivots`
 
 **hasobjectpredicate**
 

doc/ref/templates/template_pivot-pivot.rst

@@ -0,0 +1,4 @@
+
+.. include:: meta-pivot-pivot.rst
+
+.. todo:: Not yet documented.

include/zotonic_notifications.hrl

@@ -290,18 +290,6 @@
 %%      Foldr over all observers.
 -record(pivot_update, {id, raw_props}).
 
-%% @doc Foldr over a resource's pivot data, after 'pivot_rsc_data' fold.
-%% Further filtering/extending of pivot data.
--record(pivot_get, {id}).
-
-%% @doc Fetch the list of ids whose title and id will be added to the pivot data of a resource
-%% Foldr over the object ids of a resource.
--record(pivot_related, {id}).
-
-%% @doc Determine which ids will have their title indexed with the resource.
-%% Foldr over the related ids of a resource.
--record(pivot_related_text_ids, {id}).
-
 %% @doc Foldr to change or add pivot fields for the main pivot table.
 %%      The rsc contains all rsc properties for this resource, including pivot properties.
 -record(pivot_fields, {id, rsc}).

modules/mod_base/templates/pivot/_block_text.tpl

@@ -0,0 +1,5 @@
+{% for b in id.blocks %}
+    {% for z_language in id.language|default:[z_language] %}
+        {{ b.header }} {{ b.body }}
+    {% endfor %}
+{% endfor %}

modules/mod_base/templates/pivot/_main_text.tpl

@@ -0,0 +1,45 @@
+{% for cat,_true in id.is_a %}
+    {{ cat }}
+    {% for z_language in id.language|default:[z_language] %}
+        {{ m.rsc[cat].title }}
+    {% endfor %}
+{% endfor %}
+
+{% for oid in id.o.author %}
+    {% catinclude "pivot/_title.tpl" oid %}
+{% endfor %}
+
+{% for z_language in id.language|default:[z_language] %}
+    {{ id.chapeau }} {{ id.summary }} {{ id.body }} {{ id.body_extra }}
+    {{ id.date_remarks }}
+{% endfor %}
+
+{{ id.address_street_1 }}
+{{ id.address_street_2 }}
+{{ id.address_city }}
+{{ id.address_state }}
+{{ id.address_postcode }}
+{{ id.address_country }}
+{% if id.address_country and m.modules.active.mod_l10n %}
+    {% for z_language in id.language|default:[z_language] %}
+        {{ m.l10n.country_name[id.address_country] }}
+    {% endfor %}
+{% endif %}
+
+{{ id.mail_street_1 }}
+{{ id.mail_street_2 }}
+{{ id.mail_city }}
+{{ id.mail_state }}
+{{ id.mail_postcode }}
+{{ id.mail_country }}
+{% if id.mail_country and m.modules.active.mod_l10n %}
+    {% for z_language in id.language|default:[z_language] %}
+        {{ m.l10n.country_name[id.mail_country] }}
+    {% endfor %}
+{% endif %}
+
+{{ id.website }}
+
+{{ id.seo_title }}
+{{ id.seo_keywords }}
+{{ id.seo_desc }}

modules/mod_base/templates/pivot/_related_ids.tpl

@@ -0,0 +1,13 @@
+{# All related resource ids, used for finding similar resources #}
+{# Prefix object ids with 'zpo' and categories with 'zpc' #}
+{% for name,os in m.edge[id] %}
+    {% for o in os %}
+        zpo{{ o.object_id }}
+    {% endfor %}
+{% endfor %}
+{% if id.content_group_id %}
+    zpo{{ id.content_group_id }}
+{% endif %}
+{% for cat,_true in id.is_a %}
+    zpc{{ m.rsc[cat].id }}
+{% endfor %}

modules/mod_base/templates/pivot/_related_text.tpl

@@ -0,0 +1,10 @@
+{% for name,os in m.edge[id] %}
+    {% if name /= 'author' and not m.rsc[name].is_object_noindex %}
+        {% for o in os %}
+            {% catinclude "pivot/_title.tpl" o.object_id %}
+        {% endfor %}
+    {% endif %}
+{% endfor %}
+{% if id.content_group_id %}
+    {% catinclude "pivot/_title.tpl" id.content_group_id %}
+{% endif %}

modules/mod_base/templates/pivot/_title.person.tpl

@@ -0,0 +1 @@
+{% if id.name_first or id.name_surname %}{{ id.name_first }} {{ id.name_surname_prefix }} {{ id.name_surname }}{% else %}{{ id.title }}{% endif %}

modules/mod_base/templates/pivot/_title.tpl

@@ -0,0 +1 @@
+{% id.title %}

modules/mod_base/templates/pivot/_title_text.tpl

@@ -0,0 +1,8 @@
+{% for z_language in id.language|default:[z_language] %}
+    {{ id.title }} {{ id.short_title }} {{ id.subtitle }}
+{% endfor %}
+
+{{ id.name_prefix }} {{ id.name_first }} {{ id.name_surname_prefix }} {{ id.name_surname }}
+
+{{ id.slug|replace:"-":" " }}
+{{ id.page_path|replace:"/":" " }}

modules/mod_base/templates/pivot/pivot.tpl

@@ -0,0 +1,59 @@
+{# Template used by z_pivot_rsc to fill the index tables for resources #}
+{# Extend this template if a category needs special pivoting (pivot uses a catinclude) #}
+
+{# Arguments:
+ # - id (integer) the resource id to be pivoted
+ # - props (property list) the prepivoted properties of the resource
+ # - z_language (atom) the preferred stemmer language
+ #}
+
+{# Blocks a..d contain the texts to be indexed, you need to render all languages #}
+{% block a %}
+    {% catinclude "pivot/_title_text.tpl" id %}
+{% endblock %}
+
+{% block b %}
+    {% catinclude "pivot/_main_text.tpl" id %}
+{% endblock %}
+
+{% block c %}
+    {% catinclude "pivot/_block_text.tpl" id %}
+{% endblock %}
+
+{% block d %}
+    {% catinclude "pivot/_related_text.tpl" id %}
+{% endblock %}
+
+{# This title is used for sorting #}
+{% block title %}
+    {{ id.title|lower }}
+{% endblock %}
+
+{# All related resource ids, used for finding similar resources #}
+{# Prefix object ids with 'zpo' and categories with 'zpc' #}
+{% block related_ids %}
+    {% catinclude "pivot/_related_ids.tpl" id %}
+{% endblock %}
+
+{# Pivot fields #}
+{% block address_street %}{{ id.address_street_1|default:mail_address_street_1 }}{% endblock %}
+{% block address_city %}{{ id.address_city|default:mail_city }}{% endblock %}
+{% block address_postcode %}{{ id.address_postcode|default:mail_postcode }}{% endblock %}
+{% block address_state %}{{ id.address_state|default:mail_state }}{% endblock %}
+{% block address_country %}{{ id.address_country|default:mail_country }}{% endblock %}
+
+{% block name_first %}{{ id.name_first }}{% endblock %}
+{% block name_surname %}{{ id.surname }}{% endblock %}
+{% block gender %}{{ id.gender }}{% endblock %}
+
+{% block date_start %}{{ id.date_start }}{% endblock %}
+{% block date_end %}{{ id.date_end }}{% endblock %}
+{% block date_start_month_day %}{{ id.date_start|date:"md" }}{% endblock %}
+{% block date_end_month_day %}{{ id.date_end|date:"md" }}{% endblock %}
+
+{% block location_lat %}{{ id.computed_location_lat|default:id.location_lat }}{% endblock %}
+{% block location_lng %}{{ id.computed_location_lng|default:id.location_lng }}{% endblock %}
+
+{# Optional date when to repivot #}
+{# Useful to repivot events when they are past their date_end, example: {{ id.date_end }} #}
+{% block date_repivot %}{% endblock %}

modules/mod_content_groups/mod_content_groups.erl

@@ -32,7 +32,6 @@
     event/2,
     observe_rsc_get/3,
     observe_rsc_delete/2,
-    observe_pivot_related/3,
     observe_rsc_update_done/2,
     observe_admin_menu/3,
     manage_schema/2
@@ -181,22 +180,6 @@ observe_admin_menu(admin_menu, Acc, Context) ->
                 visiblecheck={acl, use, mod_admin_config}}
      |Acc].
 
-observe_pivot_related(#pivot_related{id=Id}, Ids, Context) ->
-    case m_rsc:p_no_acl(Id, content_group_id, Context) of
-        undefined ->
-            Ids;
-        CId ->
-            lists:foldl(
-                fun(PId,Acc) ->
-                    case lists:member(PId, Acc) of
-                        true -> Acc;
-                        false -> [PId|Acc]
-                    end
-                end,
-                Ids,
-                [CId | m_hierarchy:parents(content_group, CId, Context) ])
-    end.
-
 observe_rsc_update_done(#rsc_update_done{pre_is_a=PreIsA, post_is_a=PostIsA}, Context) ->
     case  lists:member('content_group', PreIsA) 
         orelse lists:member('content_group', PostIsA)