circulation: anonymize loans by tasks

A new daily job in place to anonymize loans for
all organisations, scheduled for execution at 7h00.

Loans are automatically anonymized after updates.

Anonymized loans are not returned for public and
pro searches.

Co-Authored-by: Aly Badr <aly.badr@rero.ch>

rero_ils/config.py

@@ -304,6 +304,12 @@ def _(x):
         'schedule': crontab(minute=0, hour=6),  # Every day at 06:00 UTC,
         'enabled': False
     },
+    'anonymize-loans': {
+        'task': ('rero_ils.modules.loans.tasks'
+                 '.loan_anonymizer'),
+        'schedule': crontab(minute=0, hour=7),  # Every day at 07:00 UTC,
+        'enabled': False
+    },
     'clear_and_renew_subscriptions': {
         'task':
             'rero_ils.modules.patrons.tasks.task_clear_and_renew_subscriptions',

rero_ils/modules/loans/api.py

@@ -187,11 +187,24 @@ def update_pickup_location(self, pickup_location_pid):
     @classmethod
     def create(cls, data, id_=None, delete_pid=True,
                dbcommit=False, reindex=False, **kwargs):
-        """Create a new ils record."""
+        """Create the loan record.
+
+        :param cls - class object
+        :param data - dictionary representing a loan record.
+        :param id_ - UUID, it would be generated if it is not given.
+        :param delete_pid - remove the pid present in the data if True,
+        :param dbcommit - commit the changes in the db after the creation.
+        :param reindex - index the record after the creation.
+        """
         data['$schema'] = current_jsonschemas.path_to_url(cls._schema)
         if delete_pid and data.get(cls.pid_field):
             del(data[cls.pid_field])
         cls._loan_build_org_ref(data)
+        # set the field to_anonymize
+        to_anonymize = False
+        data['to_anonymize'] = \
+            cls.can_anonymize(data) and not data.get('to_anonymize')
+
         record = super(Loan, cls).create(
             data=data, id_=id_, delete_pid=delete_pid, dbcommit=dbcommit,
             reindex=reindex, **kwargs)
@@ -200,9 +213,23 @@ def create(cls, data, id_=None, delete_pid=True,
     def update(self, data, dbcommit=False, reindex=False):
         """Update loan record."""
         self._loan_build_org_ref(data)
+        # set the field to_anonymize
+        if Loan.can_anonymize(data) and not self.get('to_anonymize'):
+            data['to_anonymize'] = True
         super(Loan, self).update(data, dbcommit, reindex)
         return self
 
+    def anonymize(self, loan, dbcommit=False, reindex=False):
+        """Anonymize a loan.
+
+        :param loan: the loan to update.
+        :param dbcommit - commit the changes in the db after the creation.
+        :param reindex - index the record after the creation.
+        """
+        loan['to_anonymize'] = True
+        super(Loan, self).update(loan, dbcommit, reindex)
+        return self
+
     def date_fields2datetime(self):
         """Convert string datetime fields to Python datetime."""
         for field in self.DATE_FIELDS + self.DATETIME_FIELDS:
@@ -419,32 +446,56 @@ def create_notification(self, notification_type=None):
             notification = notification.dispatch()
         return notification
 
-    @property
-    def concluded(self):
+    @classmethod
+    def concluded(cls, loan):
         """Check if loan is concluded.
 
         Loan is considered concluded if it has either ITEM_RETURNED or
         CANCELLED states and has no open patron_transactions.
 
+        :param loan: the loan to check.
         :return True|False
         """
         states = [LoanState.ITEM_RETURNED, LoanState.CANCELLED]
         return (
-            self.get('state') in states and
-            not loan_has_open_events(loan_pid=self.pid)
+            loan.get('state') in states and
+            not loan_has_open_events(loan_pid=loan.get('pid'))
         )
 
-    def can_anonymize(self):
-        """Check if a loan can be anonymized  and excluded from loan searches.
+    @classmethod
+    def age(cls, loan):
+        """Return the age of a loan in days.
+
+        The age of a loan is calculated based on the loan transaction date.
 
-        Loan can be anonymized if its patron has the keep_history set to False
-        and the loan is concluded.
+        :param loan: the loan to check.
+        :return loan_age in number of days
+        """
+        transaction_date = ciso8601.parse_datetime(
+            loan.get('transaction_date'))
+        loan_age = (transaction_date.replace(tzinfo=None) - datetime.utcnow())
+        return loan_age.days
 
+    @classmethod
+    def can_anonymize(cls, loan_data):
+        """Check if a loan can be anonymized and excluded from loan searches.
+
+        Loan can be anonymized if:
+        1. it is concluded and 6 months old
+        2. patron has the keep_history set to False and the loan is concluded.
+
+        This method is classmethod because it needs to check the loan record
+        during the loan.update process. this way, you can have access to the
+        old and new version of the loan.
+
+        :param loan_data: the loan to check.
         :return True|False.
         """
+        if cls.concluded(loan_data) and cls.age(loan_data) > 6*365/12:
+            return True
         keep_history = Patron.get_record_by_pid(
-            self.patron_pid).keep_history
-        return not keep_history and self.concluded
+            loan_data.get('patron_pid')).keep_history
+        return not keep_history and cls.concluded(loan_data)
 
 
 def get_request_by_item_pid_by_patron_pid(item_pid, patron_pid):
@@ -518,9 +569,10 @@ def get_loans_stats_by_patron_pid(patron_pid):
     return stats
 
 
-def get_loans_by_patron_pid(patron_pid, filter_states=[]):
+def get_loans_by_patron_pid(patron_pid, filter_states=[], to_anonymize=False):
     """Search all loans for patron to the given filter_states.
 
+    :param to_anonymize: filter by field to_anonymize.
     :param patron_pid: The patron pid.
     :param filter_states: loan states to use as a filter.
     :return: loans for given patron.
@@ -531,6 +583,7 @@ def get_loans_by_patron_pid(patron_pid, filter_states=[]):
         .params(preserve_order=True)\
         .sort({'_created': {'order': 'asc'}})\
         .source(['pid'])
+    search = search.filter('term', to_anonymize=to_anonymize)
     for loan in search.scan():
         yield Loan.get_record_by_pid(loan.pid)
 
@@ -550,7 +603,7 @@ def patron_profile(patron):
     requests = []
     history = []
 
-    for loan in get_loans_by_patron_pid(patron_pid):
+    for loan in get_loans_by_patron_pid(patron_pid, to_anonymize=False):
         item = Item.get_record_by_pid(loan.item_pid, with_deleted=True)
         if item == {}:
             # loans for deleted items are temporarily skipped.
@@ -822,6 +875,44 @@ def loan_has_open_events(loan_pid=None):
     return False
 
 
+def get_non_anonymized_loans(patron_pid=None, org_pid=None):
+    """Search all loans for non anonymized loans.
+
+    :param patron_pid: optional parameter to filter by patron_pid.
+    :param org_pid: optional parameter to filter by organisation.
+    :return: loans.
+    """
+    search = current_circulation.loan_search_cls()\
+        .filter('term', to_anonymize=False)\
+        .filter('terms', state=[LoanState.CANCELLED, LoanState.ITEM_RETURNED])\
+        .source(['pid'])
+    if patron_pid:
+        search = search.filter('term', patron__pid=patron_pid)
+    if org_pid:
+        search = search.filter('term', organisation__pid=org_pid)
+    for record in search.scan():
+        yield Loan.get_record_by_pid(record.pid)
+
+
+def anonymize_loans(
+        patron_pid=None, org_pid=None, dbcommit=False, reindex=False):
+    """Anonymise loans.
+
+    :param dbcommit - commit the changes in the db after the creation.
+    :param reindex - index the record after the creation.
+    :param patron_pid: optional parameter to filter by patron_pid.
+    :param org_pid: optional parameter to filter by organisation.
+    :return: loans.
+    """
+    counter = 0
+    for loan in get_non_anonymized_loans(
+            patron_pid=patron_pid, org_pid=org_pid):
+        if Loan.can_anonymize(loan):
+            loan.anonymize(loan, dbcommit=dbcommit, reindex=reindex)
+            counter += 1
+    return counter
+
+
 class LoansIndexer(IlsRecordsIndexer):
     """Holdings indexing class."""
 

rero_ils/modules/loans/permissions.py

@@ -45,6 +45,9 @@ def read(cls, user, record):
         :param record: Record to check.
         :return: True is action can be done.
         """
+        # Read is denied for to_anonymize loans.
+        if record.get('to_anonymize'):
+            return False
         if current_patron \
            and current_organisation.pid == Loan(record).organisation_pid:
             # staff member (lib, sys_lib) can always read loans

rero_ils/modules/loans/tasks.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+#
+# RERO ILS
+# Copyright (C) 2020 RERO
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Celery tasks for loan records."""
+
+from __future__ import absolute_import, print_function
+
+from celery import shared_task
+
+from ..loans.api import anonymize_loans
+from ..organisations.api import Organisation
+
+
+@shared_task(ignore_result=True)
+def loan_anonymizer(dbcommit=True, reindex=True):
+    """Job to anonymize loans for all organisations.
+
+    :param reindex: reindex the records.
+    :param dbcommit: commit record to database.
+    :return a count of updated loans.
+    """
+    loans_count = 0
+    for org_pid in Organisation.get_all_pids():
+        loans_count_org = anonymize_loans(
+            org_pid=org_pid, dbcommit=dbcommit, reindex=reindex)
+        loans_count = loans_count + loans_count_org
+    msg = 'number_of_loans_anonymized: {loans_count}'.format(
+            loans_count=loans_count
+    )
+
+    return msg

rero_ils/query.py

@@ -198,16 +198,21 @@ def loans_search_factory(self, search, query_parser=None):
     """Loan search factory.
 
     Restricts results to organisation level for librarian and sys_lib.
-    Restricts results to his loans for users with role patron.
+    Restricts results to its loans for users with role patron.
+    Exclude to_anonymize loans from results.
     """
     search, urlkwargs = search_factory(self, search)
+
     if current_patron:
         if current_patron.is_librarian:
             search = search.filter(
                 'term', organisation__pid=current_organisation.pid
             )
         elif current_patron.is_patron:
             search = search.filter('term', patron__pid=current_patron.pid)
+    # exclude to_anonymize records
+    search = search.filter('bool', must_not=[Q('term', to_anonymize=True)])
+
     return search, urlkwargs
 
 

scripts/setup

@@ -440,7 +440,7 @@ then
     info_msg "Start OAI harvesting asynchrone"
     eval ${PREFIX} invenio oaiharvester harvest -n ebooks -q -k
 else
-    eval ${PREFIX} invenio scheduler enable_tasks -n bulk-indexer -n claims-creation -n notification-creation -n accounts -n clear_and_renew_subscriptions -v
+    eval ${PREFIX} invenio scheduler enable_tasks -n bulk-indexer -n anonymize-loans -n claims-creation -n notification-creation -n accounts -n clear_and_renew_subscriptions -v
     info_msg "For ebooks harvesting run:"
     msg "\tinvenio oaiharvester harvest -n ebooks -a max=100 -q"
 fi

setup.py

@@ -299,6 +299,7 @@ def run(self):
             'patrons = rero_ils.modules.patrons.tasks',
             'rero_ils_collections = rero_ils.modules.collections.tasks',
             'claims = rero_ils.modules.items.tasks',
+            'loans = rero_ils.modules.loans.tasks',
         ],
         'invenio_records.jsonresolver': [
             'acq_accounts = rero_ils.modules.acq_accounts.jsonresolver',

tests/api/loans/test_loans_api_rest.py

@@ -19,10 +19,11 @@
 
 from __future__ import absolute_import, print_function
 
+from flask import url_for
 from invenio_accounts.testutils import login_user_via_session
-from utils import postdata
+from utils import get_json, postdata
 
-from rero_ils.modules.loans.api import LoanAction, patron_profile
+from rero_ils.modules.loans.api import Loan, LoanAction, patron_profile
 
 
 def test_patron_profile_loans(
@@ -64,6 +65,40 @@ def test_patron_profile_loans(
     # Some history are created
     assert patron_profile(patron_martigny_no_email)[3][0]['pid'] == loan_pid
 
+    login_user_via_session(client, librarian_martigny_no_email.user)
+    record_url = url_for(
+        'invenio_records_rest.loanid_item', pid_value=loan_pid)
+    res = client.get(record_url)
+    assert res.status_code == 200
+    assert get_json(res)['metadata']['pid'] == loan_pid
+
+    query = 'pid:{}'.format(loan_pid)
+    loan_list = url_for('invenio_records_rest.loanid_list', q=query)
+    res = client.get(loan_list)
+    assert res.status_code == 200
+    assert get_json(res)['hits']['hits'][0]['metadata']['pid'] == loan_pid
+
+    # anonymised loans are not returned.
+    loan = Loan.get_record_by_pid(loan_pid)
+    loan['to_anonymize'] = True
+    loan.update(loan, dbcommit=True, reindex=True)
+    assert not patron_profile(patron_martigny_no_email)[3]
+
+    # anonymised loans are not readable.
+    record_url = url_for(
+        'invenio_records_rest.loanid_item', pid_value=loan_pid)
+    res = client.get(record_url)
+    assert res.status_code == 403
+
+    loan_list = url_for('invenio_records_rest.loanid_list', q=query)
+    res = client.get(loan_list)
+    assert res.status_code == 200
+    assert not get_json(res)['hits']['hits']
+
     # no history is returned for deleted items.
+    loan = Loan.get_record_by_pid(loan_pid)
+    loan['to_anonymize'] = False
+    loan.update(loan, dbcommit=True, reindex=True)
+
     item_lib_martigny.delete(dbcommit=True, delindex=True)
     assert not patron_profile(patron_martigny_no_email)[3]