reports: create inventory list based on item resource

This part implements backend to perform inventory list reports in CSV format.

* Adds CSVSerializer to render search results in CSV.
* Adds ItemJSONSerializer to post process item search.
* Adapts configuration of item rest endpoint to add serializer.
* Adds library to elasticsearch mapping for item.
* Adapts listener to add library_pid into item before indexing.
* Writes tests.

Co-Authored-by: Lauren-D <laurent.dubois@itld-solutions.be>

rero_ils/config.py

@@ -464,15 +464,26 @@ def _(x):
         record_serializers={
             'application/json': (
                 'rero_ils.modules.serializers:json_v1_response'
+            ),
+            'application/rero+json': (
+                'rero_ils.modules.items.serializers:json_item_search'
             )
         },
-        record_serializers_aliases={
+        search_serializers_aliases={
             'json': 'application/json',
+            'rero+json': 'application/rero+json',
+            'csv': 'text/csv',
         },
         search_serializers={
             'application/json': (
                 'rero_ils.modules.serializers:json_v1_search'
-            )
+            ),
+            'application/rero+json': (
+                'rero_ils.modules.items.serializers:json_item_search'
+            ),
+            'text/csv': (
+                'rero_ils.modules.items.serializers:csv_item_search'
+            ),
         },
         list_route='/items/',
         record_loaders={
@@ -1272,6 +1283,36 @@ def _(x):
             _('status'): and_term_filter('holdings.items.status'),
         }
     ),
+    items=dict(
+        aggs=dict(
+            library=dict(
+                terms=dict(
+                    field='library.pid',
+                    size=RERO_ILS_DEFAULT_AGGREGATION_SIZE)
+            ),
+            location=dict(
+                terms=dict(
+                    field='location.pid',
+                    size=RERO_ILS_DEFAULT_AGGREGATION_SIZE)
+            ),
+            item_type=dict(
+                terms=dict(
+                    field='item_type.pid',
+                    size=RERO_ILS_DEFAULT_AGGREGATION_SIZE)
+            ),
+            status=dict(
+                terms=dict(
+                    field='status',
+                    size=RERO_ILS_DEFAULT_AGGREGATION_SIZE)
+            )
+        ),
+        filters={
+            _('location'): and_term_filter('location.pid'),
+            _('library'): and_term_filter('library.pid'),
+            _('item_type'): and_term_filter('item_type.pid'),
+            _('status'): and_term_filter('status')
+        },
+    ),
     patrons=dict(
         aggs=dict(
             roles=dict(

rero_ils/modules/documents/api.py

@@ -57,6 +57,15 @@ class Meta:
         doc_types = None
 
 
+def search_document_by_pid(pid):
+    """Retrieve document by pid from index."""
+    query = DocumentsSearch().filter('term', pid=pid)
+    try:
+        return next(query.scan())
+    except StopIteration:
+        return None
+
+
 class Document(IlsRecord):
     """Document class."""
 

rero_ils/modules/items/api/__init__.py

@@ -18,12 +18,13 @@
 """Item data module."""
 
 from .api import Item, ItemsIndexer, ItemsSearch, item_id_fetcher, \
-    item_id_minter
+    item_id_minter, search_active_loans_for_item
 from .circulation import ItemCirculation
 from .issue import ItemIssue
 from .record import ItemRecord
 
 __all__ = (
     'Item', 'ItemRecord', 'ItemCirculation', 'ItemIssue', 'ItemsSearch',
-    'ItemsIndexer', 'item_id_fetcher', 'item_id_minter'
+    'ItemsIndexer', 'item_id_fetcher', 'item_id_minter',
+    'search_active_loans_for_item'
 )

rero_ils/modules/items/api/api.py

@@ -20,6 +20,8 @@
 from functools import partial
 
 from elasticsearch.exceptions import NotFoundError
+from flask import current_app
+from invenio_circulation.search.api import search_by_pid
 from invenio_search import current_search
 
 from .circulation import ItemCirculation
@@ -44,6 +46,23 @@
 item_id_fetcher = partial(id_fetcher, provider=ItemProvider)
 
 
+def search_active_loans_for_item(item_pid):
+    """Return count and all active loans for an item."""
+    states = ['PENDING'] + \
+        current_app.config['CIRCULATION_STATES_LOAN_ACTIVE']
+    search = search_by_pid(
+        item_pid=item_pid,
+        filter_states=states,
+        sort_by_field='transaction_date',
+        sort_order='desc'
+    )
+    loans_count = search.count()
+    try:
+        return loans_count, search.scan()
+    except StopIteration:
+        return loans_count
+
+
 class ItemsSearch(IlsRecordsSearch):
     """ItemsSearch."""
 

rero_ils/modules/items/listener.py

@@ -37,4 +37,8 @@ def enrich_item_data(sender, json=None, record=None, index=None,
         json['organisation'] = {
             'pid': org_pid
         }
+        lib_pid = item.get_library().replace_refs()['pid']
+        json['library'] = {
+            'pid': lib_pid
+        }
         json['available'] = item.available

rero_ils/modules/items/mappings/v6/items/item-v0.0.1.json

@@ -31,6 +31,13 @@
             }
           }
         },
+        "library": {
+          "properties": {
+            "pid": {
+              "type": "keyword"
+            }
+          }
+        },
         "document": {
           "properties": {
             "pid": {

rero_ils/modules/items/serializers/__init__.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+#
+# RERO ILS
+# Copyright (C) 2019 RERO
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Items serializers."""
+
+from invenio_records_rest.serializers.response import record_responsify, \
+    search_responsify
+
+from rero_ils.modules.serializers import JSONSerializer, RecordSchemaJSONV1
+
+from .csv import ItemCSVSerializer
+from .json import ItemsJSONSerializer
+
+csv_item = ItemCSVSerializer(
+    JSONSerializer,
+    csv_included_fields=[
+        'pid',
+        'document_pid',
+        'document_title',
+        'document_type',
+        'location_name',
+        'barcode',
+        'call_number',
+        'second_call_number',
+        'loans_count',
+        'last_transaction_date',
+        'status',
+        'created'
+    ]
+)
+csv_item_response = record_responsify(csv_item, "text/csv")
+csv_item_search = search_responsify(csv_item, "text/csv")
+"""CSV serializer."""
+
+json_item = ItemsJSONSerializer(RecordSchemaJSONV1)
+"""JSON serializer."""
+
+json_item_search = search_responsify(json_item, 'application/rero+json')
+json_item_response = record_responsify(json_item, 'application/rero+json')

rero_ils/modules/items/serializers/csv.py

@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+#
+# RERO ILS
+# Copyright (C) 2019 RERO
+# Copyright (C) 2020 UCLouvain
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Item serializers."""
+
+import csv
+
+from invenio_i18n.ext import current_i18n
+from invenio_records_rest.serializers.csv import CSVSerializer, Line
+
+from rero_ils.filter import format_date_filter
+from rero_ils.modules.documents.api import search_document_by_pid
+from rero_ils.modules.documents.utils import title_format_text_head
+from rero_ils.modules.items.api import search_active_loans_for_item
+from rero_ils.modules.locations.api import LocationsSearch
+
+
+class ItemCSVSerializer(CSVSerializer):
+    """Serialize and filter item circulation status."""
+
+    def transform_search_hit(
+        self, pid, record_hit, links_factory=None, **kwargs
+    ):
+        """Transform search result hit into an intermediate representation.
+
+        :param pid: Persistent identifier instance.
+        :param pid: Persistent identifier instance.
+        :param record_hit: Record metadata retrieved via search.
+        :param links_factory: Factory function for record links.
+        """
+        hit = self.preprocess_search_hit(
+            pid, record_hit, links_factory=links_factory, **kwargs
+        )
+        return hit
+
+    def preprocess_search_hit(self, pid, record_hit, links_factory=None,
+                              **kwargs):
+        """Prepare a record hit from Elasticsearch for serialization.
+
+        :param pid: Persistent identifier instance.
+        :param record_hit: Record metadata retrieved via search.
+        :param links_factory: Factory function for record links.
+        """
+        record = record_hit['_source']
+        item_pid = pid.pid_value
+
+        # process location
+        locations_map = kwargs.get('locations_map')
+        record['location_name'] = locations_map[
+            record.get('location').get('pid')]
+
+        # retrieve and process document
+        document = search_document_by_pid(record['document']['pid'])
+        record['document_title'] = title_format_text_head(document.title,
+                                                          with_subtitle=True)
+        record['document_type'] = document.type
+
+        # get loans information
+        loans_count, loans = search_active_loans_for_item(item_pid)
+        record['loans_count'] = loans_count
+        if loans_count:
+            # get first loan
+            loan = next(loans)
+            record['last_transaction_date'] = format_date_filter(
+                loan.transaction_date,
+                format='short_date',
+                locale=current_i18n.locale.language,
+            )
+
+        record['created'] = format_date_filter(
+            record['_created'],
+            format='short_date',
+            locale=current_i18n.locale.language,
+        )
+
+        # prevent csv key error
+        # TODO: find other way to prevent csv key error
+        del(record['type'])
+
+        return record
+
+    def serialize_search(self, pid_fetcher, search_result, links=None,
+                         item_links_factory=None):
+        """Serialize a search result.
+
+        :param pid_fetcher: Persistent identifier fetcher.
+        :param search_result: Elasticsearch search result.
+        :param links: Dictionary of links to add to response.
+        :param item_links_factory: Factory function for record links.
+        """
+        records = []
+        locations_map = {}
+        for location in LocationsSearch().filter().scan():
+            locations_map[location.pid] = location.name
+        for hit in search_result['hits']['hits']:
+            processed_hit = self.transform_search_hit(
+                pid_fetcher(hit['_id'], hit['_source']),
+                hit,
+                links_factory=item_links_factory,
+                locations_map=locations_map
+            )
+            records.append(self.process_dict(processed_hit))
+
+        return self._format_csv(records)
+
+    def _format_csv(self, records):
+        """Return the list of records as a CSV string.
+
+        :param records: Records metadata to format.
+        """
+        # build a unique list of all keys in included fields as CSV headers
+        headers = dict.fromkeys(self.csv_included_fields)
+        # write the CSV output in memory
+        line = Line()
+        writer = csv.DictWriter(line,
+                                quoting=csv.QUOTE_ALL,
+                                fieldnames=headers)
+        writer.writeheader()
+        yield line.read()
+
+        for record in records:
+            writer.writerow(record)
+            yield line.read()