Merge 5fb15c2 into 34c3793

MANIFEST.in

@@ -109,3 +109,5 @@ recursive-include data *.crt
 recursive-include sonar *.babelrc
 recursive-include sonar *.eslintignore
 recursive-include tests *.json
+
+recursive-include sonar *.txt

Pipfile

@@ -6,7 +6,7 @@ name = "pypi"
 [packages]
 Babel = ">=2.4.0"
 Flask-BabelEx = ">=0.9.3"
-invenio = {version = "==3.1.0",extras = ["base", "metadata", "postgresql", "auth", "elasticsearch6" ]}
+invenio = {version = "==3.2.0a9",extras = ["base", "metadata", "files", "postgresql", "auth", "elasticsearch6" ]}
 # TODO: remove this contraint once it is solved in invenio
 raven = {version = ">=6.0",extras = ["flask"]}
 uwsgi = ">=2.0"
@@ -17,6 +17,8 @@ orcid = "*"
 python-slugify = "*"
 python3-saml = "*"
 xmltodict = "*"
+marshmallow = "<=3.0.0b6"
+pycountry = "*"
 
 [dev-packages]
 Flask-Debugtoolbar = ">=0.10.1"
@@ -29,7 +31,7 @@ marshmallow = ">=2.15.1,<3.0.0"
 pydocstyle = ">=2.0.0"
 pytest = ">=3.3.1"
 pytest-cov = ">=2.5.1"
-pytest-invenio = ">=1.0.2,<1.1.0"
+pytest-invenio = ">=1.2.1,<1.3.0"
 pytest-mock = ">=1.6.0"
 pytest-pep8 = ">=1.0.6"
 pytest-random-order = ">=0.5.4"

docker-services.yml

@@ -82,6 +82,9 @@ services:
       memlock:
         soft: -1
         hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
     mem_limit: 1g
     ports:
       - "9200:9200"
@@ -91,7 +94,7 @@ services:
     environment:
       - "ELASTICSEARCH_HOSTS=http://es:9200"
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-    ports: 
+    ports:
       - "5601:5601"
   flower:
     image: mher/flower

scripts/setup

@@ -39,4 +39,5 @@ pipenv run invenio access allow admin-access role moderator
 pipenv run invenio fixtures institutions import
 pipenv run invenio fixtures users import $(pipenv --where)/data/users.json
 pipenv run invenio fixtures documents import hevs
-pipenv run invenio fixtures documents import usi
+pipenv run invenio fixtures documents import usi
+pipenv run invenio fixtures deposits create

setup.py

@@ -64,7 +64,8 @@
         ],
         'invenio_base.api_blueprints': [
             'sonar = sonar.theme.api_views:blueprint',
-            'pdf_extractor = sonar.modules.pdf_extractor.views.api:blueprint'
+            'pdf_extractor = sonar.modules.pdf_extractor.views.api:blueprint',
+            'deposits = sonar.modules.deposits.rest:blueprint'
         ],
         'invenio_assets.webpack': [
             'sonar_theme = sonar.theme.webpack:theme'
@@ -92,31 +93,38 @@
         'invenio_jsonschemas.schemas': [
             'documents = sonar.modules.documents.jsonschemas',
             'institutions = sonar.modules.institutions.jsonschemas',
-            'users = sonar.modules.users.jsonschemas'
+            'users = sonar.modules.users.jsonschemas',
+            'deposits = sonar.modules.deposits.jsonschemas'
         ],
         'invenio_search.mappings': [
             'documents = sonar.modules.documents.mappings',
             'institutions = sonar.modules.institutions.mappings',
-            'users = sonar.modules.users.mappings'
+            'users = sonar.modules.users.mappings',
+            'deposits = sonar.modules.deposits.mappings'
         ],
         'invenio_pidstore.minters': [
             'document_id = \
                 sonar.modules.documents.api:document_pid_minter',
             'institution_id = \
                 sonar.modules.institutions.api:institution_pid_minter',
             'user_id = \
-                sonar.modules.users.api:user_pid_minter'
+                sonar.modules.users.api:user_pid_minter',
+            'deposit_id = \
+                sonar.modules.deposits.api:deposit_pid_minter'
         ],
         'invenio_pidstore.fetchers': [
             'document_id = \
                 sonar.modules.documents.api:document_pid_fetcher',
             'institution_id = \
                 sonar.modules.institutions.api:institution_pid_fetcher',
             'user_id = \
-                sonar.modules.users.api:user_pid_fetcher'
+                sonar.modules.users.api:user_pid_fetcher',
+            'deposit_id = \
+                sonar.modules.deposits.api:deposit_pid_fetcher'
         ],
         "invenio_records.jsonresolver": [
-            "institution = sonar.modules.institutions.jsonresolvers"
+            "institution = sonar.modules.institutions.jsonresolvers",
+            "user = sonar.modules.users.jsonresolvers"
         ]
     },
     classifiers=[
@@ -129,11 +137,5 @@
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.6',
         'Development Status :: 3 - Alpha',
-    ],
-    setup_requires=[
-        'pytest-runner>=3.0.0,<5',
-    ],
-    tests_require=[
-        'pytest-invenio>=1.0.0,<1.1.0',
     ]
 )

sonar/config.py

@@ -32,6 +32,7 @@
 from invenio_oauthclient.contrib import orcid
 from invenio_records_rest.facets import terms_filter
 
+from sonar.modules.deposits.api import DepositRecord, DepositSearch
 from sonar.modules.documents.api import DocumentRecord, DocumentSearch
 from sonar.modules.institutions.api import InstitutionRecord, InstitutionSearch
 from sonar.modules.permissions import can_create_record_factory, \
@@ -252,7 +253,8 @@ def _(x):
 """Records UI for sonar."""
 
 RECORDS_REST_ENDPOINTS = {
-    'doc': dict(
+    'doc':
+    dict(
         pid_type='doc',
         pid_minter='document_id',
         pid_fetcher='document_id',
@@ -283,9 +285,9 @@ def _(x):
         read_permission_factory_imp=can_read_record_factory,
         update_permission_factory_imp=can_update_record_factory,
         delete_permission_factory_imp=can_delete_record_factory,
-        list_permission_factory_imp=can_list_record_factory
-    ),
-    'inst': dict(
+        list_permission_factory_imp=can_list_record_factory),
+    'inst':
+    dict(
         pid_type='inst',
         pid_minter='institution_id',
         pid_fetcher='institution_id',
@@ -316,9 +318,9 @@ def _(x):
         read_permission_factory_imp=can_read_record_factory,
         update_permission_factory_imp=can_update_record_factory,
         delete_permission_factory_imp=can_delete_record_factory,
-        list_permission_factory_imp=can_list_record_factory
-    ),
-    'user': dict(
+        list_permission_factory_imp=can_list_record_factory),
+    'user':
+    dict(
         pid_type='user',
         pid_minter='user_id',
         pid_fetcher='user_id',
@@ -349,8 +351,37 @@ def _(x):
         read_permission_factory_imp=can_read_record_factory,
         update_permission_factory_imp=can_update_record_factory,
         delete_permission_factory_imp=can_delete_record_factory,
-        list_permission_factory_imp=can_list_record_factory
-    )
+        list_permission_factory_imp=can_list_record_factory),
+    'depo':
+    dict(
+        pid_type='depo',
+        pid_minter='deposit_id',
+        pid_fetcher='deposit_id',
+        default_endpoint_prefix=True,
+        record_class=DepositRecord,
+        search_class=DepositSearch,
+        indexer_class=RecordIndexer,
+        search_index='deposits',
+        search_type=None,
+        record_serializers={
+            'application/json': ('invenio_records_rest.serializers'
+                                 ':json_v1_response'),
+        },
+        search_serializers={
+            'application/json': ('invenio_records_rest.serializers'
+                                 ':json_v1_search'),
+        },
+        list_route='/deposits/',
+        item_route='/deposits/<pid(depo, record_class="sonar.modules.deposits'\
+        '.api:DepositRecord"):pid_value>',
+        default_media_type='application/json',
+        max_result_window=10000,
+        error_handlers=dict(),
+        create_permission_factory_imp=can_create_record_factory,
+        read_permission_factory_imp=can_read_record_factory,
+        update_permission_factory_imp=can_update_record_factory,
+        delete_permission_factory_imp=can_delete_record_factory,
+        list_permission_factory_imp=can_list_record_factory)
 }
 """REST endpoints."""
 
@@ -368,6 +399,14 @@ def _(x):
             _('author'): terms_filter('facet_authors'),
             _('subject'): terms_filter('facet_subjects'),
         }
+    ),
+    'deposits': dict(
+        aggs=dict(
+            status=dict(terms=dict(field='status'))
+        ),
+        filters={
+            _('status'): terms_filter('status')
+        }
     )
 }
 """REST search facets."""
@@ -398,6 +437,12 @@ def _(x):
 )
 """Set default sorting options."""
 
+RECORDS_FILES_REST_ENDPOINTS = {
+    'RECORDS_REST_ENDPOINTS': {
+        'depo': '/files'
+    }
+}
+
 SONAR_ENDPOINTS_ENABLED = True
 """Enable/disable automatic endpoint registration."""
 
@@ -464,3 +509,6 @@ def _(x):
 REST_ENABLE_CORS = True
 """Enable CORS to make it possible to do request to API from other
 applications."""
+
+FILES_REST_PERMISSION_FACTORY = \
+    'sonar.modules.permissions.files_permission_factory'

sonar/modules/api.py

@@ -17,6 +17,7 @@
 
 """API for manipulating records."""
 
+import re
 from uuid import uuid4
 
 from flask import current_app
@@ -25,12 +26,12 @@
 from invenio_jsonschemas import current_jsonschemas
 from invenio_pidstore.errors import PIDDoesNotExistError
 from invenio_pidstore.models import PersistentIdentifier
-from invenio_records.api import Record
+from invenio_records_files.api import FilesMixin, Record
 from invenio_search.api import RecordsSearch
 from sqlalchemy.orm.exc import NoResultFound
 
 
-class SonarRecord(Record):
+class SonarRecord(Record, FilesMixin):
     """SONAR Record."""
 
     minter = None
@@ -40,7 +41,12 @@ class SonarRecord(Record):
     schema = None
 
     @classmethod
-    def create(cls, data, id_=None, dbcommit=False, **kwargs):
+    def create(cls,
+               data,
+               id_=None,
+               dbcommit=False,
+               with_bucket=False,
+               **kwargs):
         """Create a new record."""
         assert cls.minter
         assert cls.schema
@@ -54,9 +60,12 @@ def create(cls, data, id_=None, dbcommit=False, **kwargs):
 
         cls.minter(id_, data)
 
-        record = super(SonarRecord, cls).create(data=data, id_=id_, **kwargs)
+        record = super(SonarRecord, cls).create(data=data,
+                                                with_bucket=with_bucket,
+                                                id_=id_,
+                                                **kwargs)
 
-        if(dbcommit):
+        if (dbcommit):
             record.dbcommit()
 
         return record
@@ -67,13 +76,10 @@ def get_record_by_pid(cls, pid, with_deleted=False):
         assert cls.provider
         try:
             persistent_identifier = PersistentIdentifier.get(
-                cls.provider.pid_type,
-                pid
-            )
-            return super(SonarRecord, cls).get_record(
-                persistent_identifier.object_uuid,
-                with_deleted=with_deleted
-            )
+                cls.provider.pid_type, pid)
+            return super(SonarRecord,
+                         cls).get_record(persistent_identifier.object_uuid,
+                                         with_deleted=with_deleted)
         except NoResultFound:
             return None
         except PIDDoesNotExistError:
@@ -83,9 +89,22 @@ def get_record_by_pid(cls, pid, with_deleted=False):
     def get_ref_link(cls, type, id):
         """Get $ref link for the given type of record."""
         return 'https://{host}/api/{type}/{id}'.format(
-            host=current_app.config.get('JSONSCHEMAS_HOST'),
-            type=type,
-            id=id)
+            host=current_app.config.get('JSONSCHEMAS_HOST'), type=type, id=id)
+
+    @classmethod
+    def get_pid_by_ref_link(cls, link):
+        """Return the PID corresponding to the ref link given."""
+        result = re.match(r'.*\/(.*)$', link)
+
+        if result is None:
+            raise Exception('{link} is not a valid ref link'.format(link=link))
+
+        return result.group(1)
+
+    @classmethod
+    def get_record_by_ref_link(cls, link):
+        """Get a record by its ref link."""
+        return cls.get_record_by_pid(cls.get_pid_by_ref_link(link))
 
     def dbcommit(self):
         """Commit changes to db."""

sonar/modules/cli.py

@@ -19,6 +19,7 @@
 
 import click
 
+from .deposits.cli import deposits
 from .documents.cli import documents
 from .institutions.cli import institutions
 from .users.cli import users
@@ -31,3 +32,4 @@ def fixtures():
 fixtures.add_command(documents)
 fixtures.add_command(institutions)
 fixtures.add_command(users)
+fixtures.add_command(deposits)

sonar/modules/config.py

@@ -19,6 +19,9 @@
 
 SONAR_APP_API_URL = 'https://localhost:5000/api/'
 
+SONAR_APP_ANGULAR_URL = 'https://localhost:5000/manage/'
+"""Link to angular integrated app root."""
+
 SONAR_APP_LANGUAGES_MAP = dict(
     fre='fr',
     ger='de',
@@ -35,3 +38,7 @@
 )
 
 SONAR_APP_ENABLE_CORS = True
+
+SONAR_APP_DISABLE_PERMISSION_CHECKS = False
+"""Disable permission checks during API calls. Useful when API is test from
+command line or progams like postman."""

sonar/modules/deposits/__init__.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+#
+# Swiss Open Access Repository
+# Copyright (C) 2019 RERO
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+"""Deposit module."""
+
+from __future__ import absolute_import, print_function