Merge 50f17f8 into 0358c59

scripts/setup

@@ -91,6 +91,10 @@ invenio index destroy --force --yes-i-know
 invenio utils es-init --force # To take templates into account
 invenio index queue init purge
 
+section "Initialize invenio queues."
+invenio queues delete
+invenio queues declare
+
 # Create admin role to restrict access
 section "Create roles for users" "info"
 invenio roles create superuser

sonar/config.py

@@ -866,7 +866,7 @@ def _(x):
 STATS_EVENTS = {
     'file-download': {
         'signal':
-        'invenio_files_rest.signals.file_downloaded',
+        'sonar.signals.file_downloaded',
         'templates':
         'invenio_stats.contrib.file_download',
         'event_builders':

sonar/ext.py

@@ -26,7 +26,8 @@
 from flask_security import user_registered
 from flask_wiki import Wiki
 from flask_wiki.markdown_ext import BootstrapExtension
-from invenio_files_rest.signals import file_deleted, file_uploaded
+from invenio_files_rest.signals import file_deleted, file_downloaded, \
+    file_uploaded
 from invenio_indexer.signals import before_record_index
 from werkzeug.datastructures import MIMEAccept
 
@@ -43,6 +44,7 @@
     RecordResource as ProjectRecordResource
 from sonar.resources.projects.service import \
     RecordService as ProjectRecordService
+from sonar.signals import file_download_proxy
 
 from . import config_sonar
 from .route_converters import OrganisationCodeConverter
@@ -100,6 +102,7 @@ def init_app(self, app):
 
         # Connect to signal sent when a file is uploaded or deleted
         file_uploaded.connect(file_uploaded_listener, weak=False)
+        file_downloaded.connect(file_download_proxy, weak=False)
         file_deleted.connect(file_deleted_listener, weak=False)
 
         # Add user's full name before record index

sonar/signals.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+#
+# Swiss Open Access Repository
+# Copyright (C) 2021 RERO
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Signals for SONAR."""
+
+from blinker import Namespace
+from flask import current_app
+
+_signals = Namespace()
+
+file_downloaded = _signals.signal('file-downloaded')
+"""File downloaded signal."""
+
+def file_download_proxy(obj):
+    """This proxy add a sender to the original signal.
+
+    TODO: this is a workaround that can be remove once invenio-stats has
+          fixed some issues.
+    """
+    file_downloaded.send(current_app._get_current_object(), obj=obj)

tests/api/documents/test_documents_files_rest.py

@@ -18,7 +18,9 @@
 """Test REST endpoint for documents."""
 
 
+from elasticsearch_dsl import Search
 from flask import url_for
+from invenio_stats.tasks import process_events
 
 
 def test_get_content(app, client, document_with_file):
@@ -28,7 +30,6 @@ def test_get_content(app, client, document_with_file):
     file_name = 'test1.pdf'
     fulltext_file_name = 'test1-pdf.txt'
     thumbnail_file_name = 'test1-pdf.jpg'
-
     # get the pdf file
     url_file_content = url_for('invenio_records_files.doc_object_api', pid_value=document_with_file.get('pid'), key=file_name)
     res = client.get(url_file_content)
@@ -133,3 +134,25 @@ def test_put_delete(app, client, document, pdf_file):
     # TODO: is it the right approach? Do we need to remove files and
     # the bucket?
     assert len(content) == 2
+
+
+def test_stats(app, client, document_with_file, es, event_queues):
+    """Test get existing stats for file downloads."""
+    app.config.update(SONAR_APP_DISABLE_PERMISSION_CHECKS=True)
+
+    file_name = 'test1.pdf'
+    # get the pdf file
+    url_file_content = url_for('invenio_records_files.doc_object_api', pid_value=document_with_file.get('pid'), key=file_name)
+    res = client.get(url_file_content)
+    assert res.status_code == 200
+    assert res.content_type == 'application/octet-stream'
+    assert res.content_length > 0
+
+    # process the task
+    process_events(['file-download'])
+
+    es.indices.refresh(index='events-stats-file-download')
+    search = Search(using=es).index('events-stats-file-download')
+
+    # should have at least one stats in the index
+    assert search.execute().hits.total.value > 0

tests/conftest.py

@@ -28,6 +28,7 @@
 from invenio_access.models import ActionUsers, Role
 from invenio_accounts.ext import hash_password
 from invenio_files_rest.models import Location
+from invenio_queues.proxies import current_queues
 from utils import MockArkServer
 
 from sonar.modules.collections.api import Record as CollectionRecord
@@ -39,6 +40,17 @@
 from sonar.proxies import sonar
 
 
+@pytest.fixture()
+def event_queues(app):
+    """Delete and declare test queues."""
+    current_queues.delete()
+    try:
+        current_queues.declare()
+        yield
+    finally:
+        current_queues.delete()
+
+
 @pytest.fixture(scope='module')
 def embargo_date():
     """Embargo date in one year from now."""
@@ -125,6 +137,11 @@ def app_config(app_config):
     app_config['SONAR_APP_ARK_NAAN'] = '99999'
     app_config['SONAR_APP_ARK_SCHEME'] = 'ark:'
     app_config['SONAR_APP_ARK_SHOULDER'] = 'ffk3'
+
+    # Celery
+    app_config['CELERY_BROKER_URL'] = 'memory://'
+    app_config['CELERY_TASK_ALWAYS_EAGER'] = True
+    app_config['CELERY_TASK_EAGER_PROPAGATES'] = True
     return app_config