flask2: in progress

Signed-off-by: Johnny Mariéthoz <Johnny.Mariethoz@rero.ch>

Dockerfile.base

@@ -30,9 +30,12 @@ RUN apt-get install --no-install-recommends -y git vim-tiny curl gcc g++ pkg-con
 RUN sed -i 's/<policy domain="coder" rights="none" pattern="PDF" \/>/<policy domain="coder" rights="read" pattern="PDF" \/>/g' /etc/ImageMagick-6/policy.xml
 RUN pip install --upgrade wheel pip poetry
 
-# # Install Node
-RUN curl -sL https://deb.nodesource.com/setup_14.x | bash -
-RUN apt-get install --no-install-recommends -y nodejs && rm -rf /var/lib/apt/lists/*
+# Install Node
+RUN apt-get update && apt-get install -y ca-certificates curl gnupg
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
+RUN apt-get update && apt-get install nodejs -y && rm -rf /var/lib/apt/lists/*
 
 # Env variables
 ENV WORKING_DIR=/invenio

docs/conf.py

@@ -81,7 +81,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:

pyproject.toml

@@ -10,84 +10,116 @@ python = ">= 3.9, <3.10"
 
 invenio-oaiharvester = {git = "https://github.com/inveniosoftware/invenio-oaiharvester.git", tag = "v1.0.0a4"}
 
-invenio-logging = { version = ">=1.3.0,<1.4.0", extras = ["sentry-sdk", "sentry"] }
-
-# Invenio 3.4 base modules. Same as invenio metadata extras without invenio-search-ui
-invenio-indexer = ">=1.2.0,<1.3.0"
-invenio-jsonschemas = ">=1.1.1,<1.2.0"
-invenio-oaiserver = ">1.4.0,<1.5.0"
-invenio-pidstore = ">=1.2.1,<1.3.0"
-invenio-records-rest = ">=1.8.0,<1.9.0"
-invenio-records-ui= ">=1.2.0,<1.3.0"
-invenio-records = ">=1.4.0,<1.7.0"
-invenio-stats = "^1.0.0a18"
-invenio-records-resources = "*"
-invenio-previewer = ">=1.3.5"
-invenio = {version = ">=3.4.0,<3.5.0", extras = ["base", "files", "postgresql", "auth", "elasticsearch7", "docs", "tests"]}
-
-uwsgi = ">=2.0"
-uwsgitop = ">=0.11"
-uwsgi-tools = ">=1.1.1"
-orcid = "*"
-python-slugify = "*"
-python3-saml = ">=1.13.0"
+# invenio-logging = { version = ">=1.3.0,<1.4.0", extras = ["sentry-sdk", "sentry"] }
+
+# Invenio 3.4 base modules. Same as invenio metadata extras invenio-search = {version = ">=2.1.0,<3.0.0", extras = ["elasticsearch7"]}
+# Invenio core modules
+invenio-app = ">=1.3.4,<1.4.0"
+invenio-base = ">=1.2.16,<1.3.0"
+invenio-cache = ">=1.1.1,<1.2.0"
+invenio-celery = ">=1.2.5,<1.3.0"
+invenio-config = ">=1.0.3,<1.1.0"
+invenio-i18n = ">=2.0.0,<3.0.0"
+invenio-db = {version = ">=1.1.0,<1.2.0", extras = ["postgresql"]}
+# Invenio base bundle
+invenio-admin = ">=1.4.0,<1.5.0"
+# invenio-assets = ">=3.0.0,<4.0.0" Ç Error: Patch file found for package semantic-ui-less which is not present at node_modules/semantic-ui-less
+invenio-assets = ">=2.0.0,<3.0.0"
+invenio-formatter = ">=2.0.0,<3.0.0"
+invenio-logging = {version = ">=2.0.0,<3.0.0"}
+invenio-mail = ">=2.0.0,<3.0.0"
+invenio-rest = ">=1.3.0,<1.4.0"
+invenio-theme = ">=2.5.7,<3.0.0"
+# Invenio auth bundle
+invenio-access = ">=2.0.0,<3.0.0"
+invenio-accounts = ">=3.0.0,<4.0.0"
+invenio-oauth2server = ">=2.0.0,<3.0.0"
+invenio-oauthclient = ">=3.0.0,<4.0.0"
+# Invenio metadata bundle
+invenio-indexer = ">=2.2.0,<3.0.0"
+invenio-jsonschemas = ">=1.1.4,<1.2.0"
+invenio-oaiserver = ">=2.2.0,<2.3.0"
+invenio-pidstore = ">=1.3.0,<1.4.0"
+invenio-records-rest = ">=2.2.0,<2.3.0"
+invenio-records-ui = ">=1.2.0,<1.3.0"
+invenio-records = ">=2.1.0,<2.3.0"
+invenio-search-ui = ">=2.4.0,<3.0.0"
+# files
+invenio-files-rest = '>=2.0.0,<3.0.0'
+invenio-previewer = '>=2.0.0,<3.0.0'
+invenio-records-files = '>=1.2.1,<1.3.0'
+
+# Pinned due to before_first_request deprecation https://flask.palletsprojects.com/en/2.2.x/api/#flask.Flask.before_first_request
+Flask = ">=2.2.0,<2.3.0"
+sentry-sdk = ">=1.0.0" # normaly in invenio-logging = {version = ">=2.0.0,<3.0.0", extras = ["sentry_sdk"]}
+
+## RERO ILS specific python modules
+PyYAML = ">=5.3.1"
+dateparser = ">=1.1.1"
+isbnlib = ">=3.9.1"
+requests = ">=2.20.0"
+polib = "*"
 xmltodict = "*"
-marshmallow = ">=3.0.0,<4.0.0"
-pycountry = "*"
+redisbeat = ">1.2.5, <1.3.0"
+jsonpickle = ">=1.4.1"
+ciso8601 = "*"
+
+## Additionnal constraints on python modules
 markdown-captions = "*"
-bleach = ">3.11"
-wand = ">=0.6.6,<0.7.0"
-python-dotenv = "*"
-flask-cors = ">3.0.8"
-cryptography = ">38.0.2,<40.0"
-netaddr = "*"
-dcxml = "*"
-lxml = ">=4.9.1,<5.0.0"
-webdavclient3 = ">=3.14.5"
-fuzzywuzzy = ">=0.18.0"
-python-Levenshtein = ">=0.12.0"
-polib = ">=1.1.0"
-idutils = ">=1.1.8"
-pillow = ">=9.0.0"
-celery = ">=5.0.0"
-wtforms = "<3.0.0"
-flask-wtf = "<1.0.0"
 zipp = "*"
-pycparser = "*"
-sqlalchemy = "<1.4.0"
-sqlalchemy_continuum = "<2.0.0"
-psycopg2-binary = "<3.0.0"
-MarkupSafe = "<2.1.0"
-jedi = "<0.18.0"
-ipython_genutils = "^0.2.0"
-Flask = "<2.0.0"
-SQLAlchemy = "<1.4.0"
-flask-wiki = "^0.2.2"
-sentry-sdk = "<1.6.1"
-pytest-invenio = ">=1.4.0,<1.4.12"
+
+## Deployment
+python-dotenv = ">=0.13.0"
+pydocstyle = ">=6.1.1"
+
+## Third party optional modules used by RERO ILS
+freezegun = "^1.1.0"
+lazyreader = ">1.0.0"
+jinja2 = ">2.11.2"
+jsonmerge = "^1.8.0"
+num2words = "^0.5.10"
+iso639 = "^0.1.4"
+dcxml = "^0.1.2"
+DeepDiff = "^5.5.0"
+docutils = "<0.18.0"
+wtforms = "<3.0.0"
+poethepoet = "^0.12.3"
+# to avoid conflict for urllib3
 dparse = ">=0.5.2"
 Mako = ">=1.2.2"
-jsonref = "<1.0.0"
-jsonresolver = "<0.3.2"
-setuptools = "<58"
-charset-normalizer = "<2.1.0"
-python-levenshtein = "<0.20.0"
-jsonschema = "<4.0.0"
-pydocstyle = ">=6.1.1,<6.2"
-requests-mock = "^1.11.0"
+rero-invenio-base = { git = "https://github.com/rero/rero-invenio-base.git", branch = "master" }
+# rero-invenio-base = "^0.2.1"
+jsonresolver = "*"
+# needed for elasticsearch 7.13.4
+urllib3 = "<2.0.0"
+pyparsing = "^3.1.1"
+flask-wiki = "^0.3.1"
+invenio-stats = "^4.0.1"
+fuzzywuzzy = "^0.18.0"
+pycountry = "^23.12.11"
+netaddr = "^0.9.0"
+wand = "^0.6.13"
+webdavclient3 = "^3.14.6"
 pysftp = "^0.2.9"
-rero-invenio-base = "^0.2.1"
+python-slugify = "*"
+orcid = "^1.0.3"
+invenio-records-resources = "^4.18.3"
+python3-saml = "^1.16.0"
+python-levenshtein = "^0.23.0"
+invenio-userprofiles = "^2.3.1"
+jsonschema = "<=4.20.0"
 
 [tool.poetry.dev-dependencies]
+pytest-invenio = ">=2.1.6,<3.0.0"
+Sphinx = ">=4.5.0"
 Flask-Debugtoolbar = ">=0.10.1"
-Sphinx = ">=3.0.0,<4"
-mock = ">=2.0.0"
-pytest-invenio = ">=1.4.1,<1.5.0"
+## RERO ILS specific python packages
 safety = ">=1.8"
+mock = ">=2.0.0"
+autoflake = ">=1.3.1"
 appnope = { version = "*", optional = true }
-autoflake = ">=1.4"
 
-[project.console_scripts]
+[tool.poetry.plugins."console_scripts"]
 sonar = "invenio_app.cli:cli"
 
 [tool.poetry.plugins."flask.commands"]
@@ -214,6 +246,10 @@ stats = "sonar.modules.stats.admin:stats_adminview"
 [tool.poetry.plugins."babel.extractors"]
 json = "sonar.modules.babel_extractors:extract_json"
 
+
+[tool.poetry.group.dev.dependencies]
+requests-mock = "^1.11.0"
+
 [tool.poe.tasks]
 bootstrap = {cmd = "./scripts/bootstrap", help = "Runs bootstrap"}
 console = {cmd = "./scripts/console", help = "Opens invenio shell"}

scripts/bootstrap

@@ -53,20 +53,10 @@ done
 
 title "Bootstrap script"
 
-# Update pip version
-section "Upgrading pip..." "info"
-poetry run pip install "pip<=23.0.0"
 
-# Install setuptools <58 needed to install fs 0.5.4
-section "Install setuptools<58" "info"
-poetry run pip install "setuptools<58"
+#section "Install poethepoet"
+#poetry run pip install poethepoet
 
-section "Install poethepoet"
-poetry run pip install poethepoet
-
-# Needed for poetry > 1.1
-section "Install fs"
-poetry run pip install "fs<2.0.0"
 
 if ! $deploy ; then
   if $ci ; then

scripts/test

@@ -29,35 +29,26 @@ if [[ -z "${VIRTUAL_ENV}" ]]; then
   exit 1
 fi
 
-# TODO: Exceptions on safety check
-# +============================+===========+==========================+==========+
-# | package                    | installed | affected                 | ID       |
-# +============================+===========+==========================+==========+
-# | wtforms                    | 2.3.3     | <3.0.0a1                 | 42852    |
-# | werkzeug                   | 1.0.1     | <2.2.3                   | 53325    |
-# | werkzeug                   | 1.0.1     | <2.2.3                   | 53326    |
-# | werkzeug                   | 1.0.1     | >=0,<2.1.1               | 54456    |
-# | sqlalchemy                 | 1.3.24    | <2.0.0b1                 | 51668    |
-# | sqlalchemy-utils           | 0.35.0    | >=0.27.0                 | 42194    |
-# | setuptools                 | 57.5.0    | <65.5.1                  | 52495    |
-# | sentry-sdk                 | 1.6.0     | <1.14.0                  | 53812    |
-# | safety                     | 1.10.3    | <2.2.0                   | 51358    |
-# | py                         | 1.11.0    | <=1.11.0                 | 51457    |
-# | nbconvert                  | 6.4.5     | <6.5.1                   | 50792    |
-# | flask-security             | 3.0.0     | <3.1.0                   | 45183    |
-# | flask-security             | 3.0.0     | >0                       | 44501    |
-# | click                      | 7.1.2     | <8.0.0                   | 47833    |
-# | celery                     | 5.1.2     | <5.2.0                   | 42498    |
-# | celery                     | 5.1.2     | <5.2.2                   | 43738    |
-# | cryptography               | 39.0.2    | <41.0.0                  | 59062    |
-# | cryptography               | 39.0.2    | <41.0.2                  | 59473    |
-# | cryptography               | 39.0.2    | >=0.8, <41.0.3           | 60224    |
-# | cryptography               | 39.0.2    | >=0.8, <41.0.3           | 60225    |
-# | cryptography               | 39.0.2    | >=0.8,<41.0.3            | 60223    |
-# | certifi                    | 2022.12.7 | >=2015.04.28,<2023.07.22 | 59956    |
-# | pillow                     | 9.5.0     | <10.0.1                  | 61489    |
-# +==============================================================================+
-safety check -i 45183 -i 44501 -i 51668 -i 42194 -i 42852 -i 53325 -i 53326 -i 54456 -i 42498 -i 43738 -i 47833 -i 51457 -i 51358 -i 53812 -i 52495 -i 50792 -i 55261 -i 59062 -i 59473 -i 59956 -i 60223 -i 60224 -i 60225 -i 61489 -i 62019 -i 40459 -i 62451 -i 62452 -i 62556
+# -> Vulnerability found in flask-caching version 2.0.1
+#    Vulnerability ID: 40459
+# -> Vulnerability found in sqlalchemy version 1.4.50
+#    Vulnerability ID: 51668
+# -> Vulnerability found in sqlalchemy-utils version 0.38.3
+#    Vulnerability ID: 42194
+# -> Vulnerability found in wtforms version 2.3.3
+#    Vulnerability ID: 42852
+# -> Vulnerability found in werkzeug version 2.2.3
+#    Vulnerability ID: 62019
+# -> Vulnerability found in py version 1.11.0
+#    Vulnerability ID: 51457
+echo "Check vulnerabilities:"
+safety_exceptions="-i 40459 -i 51668 -i 42194 -i 42852 -i 62019 -i 51457"
+msg=$(safety check -o text ${safety_exceptions}) || {
+  echo "Safety vulnerabilites found for packages:" $(safety check -o bare ${safety_exceptions})
+  echo "Run:" "safety check -o screen ${safety_exceptions} | grep -i vulnerability" "for more details"
+  exit 1
+}
+
 pydocstyle sonar tests docs
 isort --check-only --diff "${SCRIPT_PATH}/.."
 autoflake -c -r --remove-all-unused-imports --ignore-init-module-imports . &> /dev/null || {

sonar/config.py

@@ -206,7 +206,9 @@ def _(x):
 # JSONSchemas
 # ===========
 #: Hostname used in URLs for local JSONSchemas.
+JSONSCHEMAS_URL_SCHEME = 'https'
 JSONSCHEMAS_HOST = 'sonar.ch'
+JSONSCHEMAS_REPLACE_REFS = True
 
 # Flask configuration
 # ===================
@@ -291,6 +293,7 @@ def _(x):
 SECURITY_EMAIL_SUBJECT_PASSWORD_RESET = _('SONAR password reset')
 SECURITY_EMAIL_SUBJECT_PASSWORD_NOTICE = _(
     'Your SONAR password has been reset')
+SECURITY_PASSWORD_SINGLE_HASH = True
 
 RECORDS_UI_ENDPOINTS = {
     'doc': {
@@ -378,7 +381,6 @@ def _(x):
         search_class=DocumentSearch,
         indexer_class='sonar.modules.documents.api:DocumentIndexer',
         search_index='documents',
-        search_type=None,
         record_serializers={
             'application/json': ('sonar.modules.documents.serializers'
                                  ':json_v1_response'),
@@ -429,7 +431,6 @@ def _(x):
         search_class=OrganisationSearch,
         indexer_class='sonar.modules.organisations.api:OrganisationIndexer',
         search_index='organisations',
-        search_type=None,
         record_serializers={
             'application/json': ('sonar.modules.organisations.serializers'
                                  ':json_v1_response'),
@@ -468,7 +469,6 @@ def _(x):
         search_class=UserSearch,
         indexer_class='sonar.modules.users.api:UserIndexer',
         search_index='users',
-        search_type=None,
         record_serializers={
             'application/json': ('sonar.modules.users.serializers'
                                  ':json_v1_response'),
@@ -507,7 +507,6 @@ def _(x):
         search_class=DepositSearch,
         indexer_class='sonar.modules.deposits.api:DepositIndexer',
         search_index='deposits',
-        search_type=None,
         record_serializers={
             'application/json': ('sonar.modules.deposits.serializers'
                                  ':json_v1_response'),

sonar/dedicated/hepvs/projects/jsonschemas/hepvs/projects/project-v1.0.0_src.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "id": "https://sonar.ch/schemas/projects/project-v1.0.0.json",
   "title": "Research project",
   "type": "object",

sonar/ext.py

@@ -141,7 +141,7 @@ def init_views(self, app):
             'view': app.config.get('SONAR_APP_DEFAULT_ORGANISATION')})
 
 
-        @app.route('/<org_code:view>/')
+        @app.route('/<org_code:view>')
         def index(view):
             """Homepage."""
             return render_template('sonar/frontpage.html', view=view)

sonar/modules/api.py

@@ -259,6 +259,13 @@ def add_file_from_url(self, url, key, **kwargs):
             kwargs['force_external_url'] = True
             self.add_file(b'', key, **kwargs)
 
+    def replace_refs(self):
+        """Replace the ``$ref`` keys within the JSON.
+
+        Note: needed for jsonref>=1.0.0
+        """
+        return (type(self))(super().replace_refs())
+
     def add_file(self, data, key, **kwargs):
         """Create file and add it to record.
 
@@ -391,7 +398,7 @@ def index(self, record):
         """
         return_value = super(SonarIndexer, self).index(record)
 
-        index_name, doc_type = current_record_to_index(record)
+        index_name = current_record_to_index(record)
         current_search.flush_and_refresh(index_name)
         return return_value
 
@@ -402,6 +409,6 @@ def delete(self, record):
         :returns: Indexation result
         """
         return_value = super(SonarIndexer, self).delete(record)
-        index_name, doc_type = current_record_to_index(record)
+        index_name = current_record_to_index(record)
         current_search.flush_and_refresh(index_name)
         return return_value