flask2: in progress

Signed-off-by: Johnny Mariéthoz <Johnny.Mariethoz@rero.ch>

docs/conf.py

@@ -81,7 +81,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:

pyproject.toml

@@ -10,84 +10,116 @@ python = ">= 3.9, <3.10"
 
 invenio-oaiharvester = {git = "https://github.com/inveniosoftware/invenio-oaiharvester.git", tag = "v1.0.0a4"}
 
-invenio-logging = { version = ">=1.3.0,<1.4.0", extras = ["sentry-sdk", "sentry"] }
-
-# Invenio 3.4 base modules. Same as invenio metadata extras without invenio-search-ui
-invenio-indexer = ">=1.2.0,<1.3.0"
-invenio-jsonschemas = ">=1.1.1,<1.2.0"
-invenio-oaiserver = ">1.4.0,<1.5.0"
-invenio-pidstore = ">=1.2.1,<1.3.0"
-invenio-records-rest = ">=1.8.0,<1.9.0"
-invenio-records-ui= ">=1.2.0,<1.3.0"
-invenio-records = ">=1.4.0,<1.7.0"
-invenio-stats = "^1.0.0a18"
-invenio-records-resources = "*"
-invenio-previewer = ">=1.3.5"
-invenio = {version = ">=3.4.0,<3.5.0", extras = ["base", "files", "postgresql", "auth", "elasticsearch7", "docs", "tests"]}
-
-uwsgi = ">=2.0"
-uwsgitop = ">=0.11"
-uwsgi-tools = ">=1.1.1"
-orcid = "*"
-python-slugify = "*"
-python3-saml = ">=1.13.0"
+# invenio-logging = { version = ">=1.3.0,<1.4.0", extras = ["sentry-sdk", "sentry"] }
+
+# Invenio 3.4 base modules. Same as invenio metadata extras invenio-search = {version = ">=2.1.0,<3.0.0", extras = ["elasticsearch7"]}
+# Invenio core modules
+invenio-app = ">=1.3.4,<1.4.0"
+invenio-base = ">=1.2.16,<1.3.0"
+invenio-cache = ">=1.1.1,<1.2.0"
+invenio-celery = ">=1.2.5,<1.3.0"
+invenio-config = ">=1.0.3,<1.1.0"
+invenio-i18n = ">=2.0.0,<3.0.0"
+invenio-db = {version = ">=1.1.0,<1.2.0", extras = ["postgresql"]}
+# Invenio base bundle
+invenio-admin = ">=1.4.0,<1.5.0"
+# invenio-assets = ">=3.0.0,<4.0.0" Ç Error: Patch file found for package semantic-ui-less which is not present at node_modules/semantic-ui-less
+invenio-assets = ">=2.0.0,<3.0.0"
+invenio-formatter = ">=2.0.0,<3.0.0"
+invenio-logging = {version = ">=2.0.0,<3.0.0"}
+invenio-mail = ">=2.0.0,<3.0.0"
+invenio-rest = ">=1.3.0,<1.4.0"
+invenio-theme = ">=2.5.7,<3.0.0"
+# Invenio auth bundle
+invenio-access = ">=2.0.0,<3.0.0"
+invenio-accounts = ">=3.0.0,<4.0.0"
+invenio-oauth2server = ">=2.0.0,<3.0.0"
+invenio-oauthclient = ">=3.0.0,<4.0.0"
+# Invenio metadata bundle
+invenio-indexer = ">=2.2.0,<3.0.0"
+invenio-jsonschemas = ">=1.1.4,<1.2.0"
+invenio-oaiserver = ">=2.2.0,<2.3.0"
+invenio-pidstore = ">=1.3.0,<1.4.0"
+invenio-records-rest = ">=2.2.0,<2.3.0"
+invenio-records-ui = ">=1.2.0,<1.3.0"
+invenio-records = ">=2.1.0,<2.3.0"
+invenio-search-ui = ">=2.4.0,<3.0.0"
+# files
+invenio-files-rest = '>=2.0.0,<3.0.0'
+invenio-previewer = '>=2.0.0,<3.0.0'
+invenio-records-files = '>=1.2.1,<1.3.0'
+
+# Pinned due to before_first_request deprecation https://flask.palletsprojects.com/en/2.2.x/api/#flask.Flask.before_first_request
+Flask = ">=2.2.0,<2.3.0"
+sentry-sdk = ">=1.0.0" # normaly in invenio-logging = {version = ">=2.0.0,<3.0.0", extras = ["sentry_sdk"]}
+
+## RERO ILS specific python modules
+PyYAML = ">=5.3.1"
+dateparser = ">=1.1.1"
+isbnlib = ">=3.9.1"
+requests = ">=2.20.0"
+polib = "*"
 xmltodict = "*"
-marshmallow = ">=3.0.0,<4.0.0"
-pycountry = "*"
+redisbeat = ">1.2.5, <1.3.0"
+jsonpickle = ">=1.4.1"
+ciso8601 = "*"
+
+## Additionnal constraints on python modules
 markdown-captions = "*"
-bleach = ">3.11"
-wand = ">=0.6.6,<0.7.0"
-python-dotenv = "*"
-flask-cors = ">3.0.8"
-cryptography = ">38.0.2,<40.0"
-netaddr = "*"
-dcxml = "*"
-lxml = ">=4.9.1,<5.0.0"
-webdavclient3 = ">=3.14.5"
-fuzzywuzzy = ">=0.18.0"
-python-Levenshtein = ">=0.12.0"
-polib = ">=1.1.0"
-idutils = ">=1.1.8"
-pillow = ">=9.0.0"
-celery = ">=5.0.0"
-wtforms = "<3.0.0"
-flask-wtf = "<1.0.0"
 zipp = "*"
-pycparser = "*"
-sqlalchemy = "<1.4.0"
-sqlalchemy_continuum = "<2.0.0"
-psycopg2-binary = "<3.0.0"
-MarkupSafe = "<2.1.0"
-jedi = "<0.18.0"
-ipython_genutils = "^0.2.0"
-Flask = "<2.0.0"
-SQLAlchemy = "<1.4.0"
-flask-wiki = "^0.2.2"
-sentry-sdk = "<1.6.1"
-pytest-invenio = ">=1.4.0,<1.4.12"
+
+## Deployment
+python-dotenv = ">=0.13.0"
+pydocstyle = ">=6.1.1"
+
+## Third party optional modules used by RERO ILS
+freezegun = "^1.1.0"
+lazyreader = ">1.0.0"
+jinja2 = ">2.11.2"
+jsonmerge = "^1.8.0"
+num2words = "^0.5.10"
+iso639 = "^0.1.4"
+dcxml = "^0.1.2"
+DeepDiff = "^5.5.0"
+docutils = "<0.18.0"
+wtforms = "<3.0.0"
+poethepoet = "^0.12.3"
+# to avoid conflict for urllib3
 dparse = ">=0.5.2"
 Mako = ">=1.2.2"
-jsonref = "<1.0.0"
-jsonresolver = "<0.3.2"
-setuptools = "<58"
-charset-normalizer = "<2.1.0"
-python-levenshtein = "<0.20.0"
-jsonschema = "<4.0.0"
-pydocstyle = ">=6.1.1,<6.2"
-requests-mock = "^1.11.0"
+rero-invenio-base = { git = "https://github.com/rero/rero-invenio-base.git", branch = "master" }
+# rero-invenio-base = "^0.2.1"
+jsonresolver = "*"
+# needed for elasticsearch 7.13.4
+urllib3 = "<2.0.0"
+pyparsing = "^3.1.1"
+flask-wiki = "^0.3.1"
+invenio-stats = "^4.0.1"
+fuzzywuzzy = "^0.18.0"
+pycountry = "^23.12.11"
+netaddr = "^0.9.0"
+wand = "^0.6.13"
+webdavclient3 = "^3.14.6"
 pysftp = "^0.2.9"
-rero-invenio-base = "^0.2.1"
+python-slugify = "*"
+orcid = "^1.0.3"
+invenio-records-resources = "^4.18.3"
+python3-saml = "^1.16.0"
+python-levenshtein = "^0.23.0"
+invenio-userprofiles = "^2.3.1"
+jsonschema = "<=4.20.0"
 
 [tool.poetry.dev-dependencies]
+pytest-invenio = ">=2.1.6,<3.0.0"
+Sphinx = ">=4.5.0"
 Flask-Debugtoolbar = ">=0.10.1"
-Sphinx = ">=3.0.0,<4"
-mock = ">=2.0.0"
-pytest-invenio = ">=1.4.1,<1.5.0"
+## RERO ILS specific python packages
 safety = ">=1.8"
+mock = ">=2.0.0"
+autoflake = ">=1.3.1"
 appnope = { version = "*", optional = true }
-autoflake = ">=1.4"
 
-[project.console_scripts]
+[tool.poetry.plugins."console_scripts"]
 sonar = "invenio_app.cli:cli"
 
 [tool.poetry.plugins."flask.commands"]
@@ -214,6 +246,10 @@ stats = "sonar.modules.stats.admin:stats_adminview"
 [tool.poetry.plugins."babel.extractors"]
 json = "sonar.modules.babel_extractors:extract_json"
 
+
+[tool.poetry.group.dev.dependencies]
+requests-mock = "^1.11.0"
+
 [tool.poe.tasks]
 bootstrap = {cmd = "./scripts/bootstrap", help = "Runs bootstrap"}
 console = {cmd = "./scripts/console", help = "Opens invenio shell"}

scripts/bootstrap

@@ -54,19 +54,19 @@ done
 title "Bootstrap script"
 
 # Update pip version
-section "Upgrading pip..." "info"
-poetry run pip install "pip<=23.0.0"
+# section "Upgrading pip..." "info"
+# poetry run pip install "pip<=23.0.0"
 
-# Install setuptools <58 needed to install fs 0.5.4
-section "Install setuptools<58" "info"
-poetry run pip install "setuptools<58"
+# # Install setuptools <58 needed to install fs 0.5.4
+# section "Install setuptools<58" "info"
+# poetry run pip install "setuptools<58"
 
 section "Install poethepoet"
 poetry run pip install poethepoet
 
-# Needed for poetry > 1.1
-section "Install fs"
-poetry run pip install "fs<2.0.0"
+# # Needed for poetry > 1.1
+# section "Install fs"
+# poetry run pip install "fs<2.0.0"
 
 if ! $deploy ; then
   if $ci ; then

sonar/config.py

@@ -206,7 +206,9 @@ def _(x):
 # JSONSchemas
 # ===========
 #: Hostname used in URLs for local JSONSchemas.
+JSONSCHEMAS_URL_SCHEME = 'https'
 JSONSCHEMAS_HOST = 'sonar.ch'
+JSONSCHEMAS_REPLACE_REFS = True
 
 # Flask configuration
 # ===================
@@ -291,6 +293,7 @@ def _(x):
 SECURITY_EMAIL_SUBJECT_PASSWORD_RESET = _('SONAR password reset')
 SECURITY_EMAIL_SUBJECT_PASSWORD_NOTICE = _(
     'Your SONAR password has been reset')
+SECURITY_PASSWORD_SINGLE_HASH = True
 
 RECORDS_UI_ENDPOINTS = {
     'doc': {
@@ -378,7 +381,6 @@ def _(x):
         search_class=DocumentSearch,
         indexer_class='sonar.modules.documents.api:DocumentIndexer',
         search_index='documents',
-        search_type=None,
         record_serializers={
             'application/json': ('sonar.modules.documents.serializers'
                                  ':json_v1_response'),
@@ -429,7 +431,6 @@ def _(x):
         search_class=OrganisationSearch,
         indexer_class='sonar.modules.organisations.api:OrganisationIndexer',
         search_index='organisations',
-        search_type=None,
         record_serializers={
             'application/json': ('sonar.modules.organisations.serializers'
                                  ':json_v1_response'),
@@ -468,7 +469,6 @@ def _(x):
         search_class=UserSearch,
         indexer_class='sonar.modules.users.api:UserIndexer',
         search_index='users',
-        search_type=None,
         record_serializers={
             'application/json': ('sonar.modules.users.serializers'
                                  ':json_v1_response'),
@@ -507,7 +507,6 @@ def _(x):
         search_class=DepositSearch,
         indexer_class='sonar.modules.deposits.api:DepositIndexer',
         search_index='deposits',
-        search_type=None,
         record_serializers={
             'application/json': ('sonar.modules.deposits.serializers'
                                  ':json_v1_response'),

sonar/dedicated/hepvs/projects/jsonschemas/hepvs/projects/project-v1.0.0_src.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "id": "https://sonar.ch/schemas/projects/project-v1.0.0.json",
   "title": "Research project",
   "type": "object",

sonar/ext.py

@@ -141,7 +141,7 @@ def init_views(self, app):
             'view': app.config.get('SONAR_APP_DEFAULT_ORGANISATION')})
 
 
-        @app.route('/<org_code:view>/')
+        @app.route('/<org_code:view>')
         def index(view):
             """Homepage."""
             return render_template('sonar/frontpage.html', view=view)

sonar/modules/api.py

@@ -259,6 +259,13 @@ def add_file_from_url(self, url, key, **kwargs):
             kwargs['force_external_url'] = True
             self.add_file(b'', key, **kwargs)
 
+    def replace_refs(self):
+        """Replace the ``$ref`` keys within the JSON.
+
+        Note: needed for jsonref>=1.0.0
+        """
+        return (type(self))(super().replace_refs())
+
     def add_file(self, data, key, **kwargs):
         """Create file and add it to record.
 
@@ -391,7 +398,7 @@ def index(self, record):
         """
         return_value = super(SonarIndexer, self).index(record)
 
-        index_name, doc_type = current_record_to_index(record)
+        index_name = current_record_to_index(record)
         current_search.flush_and_refresh(index_name)
         return return_value
 
@@ -402,6 +409,6 @@ def delete(self, record):
         :returns: Indexation result
         """
         return_value = super(SonarIndexer, self).delete(record)
-        index_name, doc_type = current_record_to_index(record)
+        index_name = current_record_to_index(record)
         current_search.flush_and_refresh(index_name)
         return return_value

sonar/modules/cli/utils.py

@@ -30,9 +30,9 @@
 from invenio_files_rest.models import Location
 from invenio_jsonschemas import current_jsonschemas
 from invenio_records_rest.utils import obj_or_import_string
-from invenio_search.cli import es_version_check
+from invenio_search.cli import search_version_check
 from invenio_search.proxies import current_search
-from jsonref import JsonLoader
+from jsonref import jsonloader
 
 from sonar.modules.api import SonarRecord
 
@@ -45,7 +45,7 @@ def utils():
 @utils.command()
 @click.option('--force', is_flag=True, default=False)
 @with_appcontext
-@es_version_check
+@search_version_check
 def es_init(force):
     """Initialize registered templates, aliases and mappings."""
     # TODO: to remove once it is fixed in invenio-search module
@@ -87,25 +87,21 @@ def compile_json(src_json_file, output):
     click.secho('Compile json file (resolve $ref): ', fg='green', nl=False)
     click.secho(src_json_file.name)
 
-    data = jsonref.load(src_json_file, loader=CustomJsonLoader())
+    data = jsonref.load(src_json_file, loader=custom_json_loader)
     if not output:
         output = sys.stdout
     json.dump(data, fp=output, indent=2)
 
 
-class CustomJsonLoader(JsonLoader):
-    """Custom JSON ref loader."""
+def custom_json_loader(uri, **kwargs):
+    """Method invoked when an uri has to be resolved.
 
-    def __call__(self, uri, **kwargs):
-        """Method invoked when an uri has to be resolved.
-
-        If URI is present in registered JSON schemas list, it resolves in the
-        common schemas, else lets the loader from jsonref do the job.
-        """
-        if uri in current_jsonschemas.list_schemas():
-            return current_jsonschemas.get_schema(uri)
-
-        return super(CustomJsonLoader, self).__call__(uri, *kwargs)
+    If URI is present in registered JSON schemas list, it resolves in the
+    common schemas, else lets the loader from jsonref do the job.
+    """
+    if uri in current_jsonschemas.list_schemas():
+        return current_jsonschemas.get_schema(uri)
+    return jsonloader(uri, *kwargs)
 
 
 @utils.command('export')

sonar/modules/collections/config.py

@@ -56,8 +56,6 @@ class Configuration:
         f'{MODULE_PATH}.api:RecordIndexer',
         'search_index':
         RESOURCE_NAME,
-        'search_type':
-        None,
         'record_serializers': {
             'application/json': (f'{MODULE_PATH}.serializers'
                                  ':json_v1_response'),

sonar/modules/collections/jsonschemas/collections/collection-v1.0.0_src.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "id": "https://sonar.ch/schemas/collections/collection-v1.0.0.json",
   "title": "Collections",
   "type": "object",