Merge d0ffdd2 into c3b073c

.github/workflows/continuous-integration-test.yml

@@ -11,7 +11,7 @@ jobs:
       matrix:
         dependencies: ['dev', 'deploy']
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Install Libraries
       run: |
@@ -25,9 +25,9 @@ jobs:
         sudo sed -i 's/<policy domain="coder" rights="none" pattern="PDF" \/>/<policy domain="coder" rights="read" pattern="PDF" \/>/g' /etc/ImageMagick-6/policy.xml
 
     - name: Setup node
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
-        node-version: '14'
+        node-version: '18'
 
     - name: Docker compose up
       run: docker-compose up -d

Dockerfile.base

@@ -30,9 +30,12 @@ RUN apt-get install --no-install-recommends -y git vim-tiny curl gcc g++ pkg-con
 RUN sed -i 's/<policy domain="coder" rights="none" pattern="PDF" \/>/<policy domain="coder" rights="read" pattern="PDF" \/>/g' /etc/ImageMagick-6/policy.xml
 RUN pip install --upgrade wheel pip poetry
 
-# # Install Node
-RUN curl -sL https://deb.nodesource.com/setup_14.x | bash -
-RUN apt-get install --no-install-recommends -y nodejs && rm -rf /var/lib/apt/lists/*
+# Install Node
+RUN apt-get update && apt-get install -y ca-certificates curl gnupg
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
+RUN apt-get update && apt-get install nodejs -y && rm -rf /var/lib/apt/lists/*
 
 # Env variables
 ENV WORKING_DIR=/invenio

data/oai_sources.json

@@ -10,7 +10,7 @@
   {
     "key": "archive_ouverte_unige",
     "name": "Archive ouverte UNIGE",
-    "url": "https://archive-ouverte.unige.ch/oaiprovider/",
+    "url": "https://archive-ouverte.unige.ch/oai",
     "metadataprefix": "marc21",
     "comment": "",
     "setspecs": ""

docker-services.yml

@@ -109,7 +109,7 @@ services:
     ports:
       - "5555:5555"
   grobid:
-    image: lfoppiano/grobid:0.7.0
+    image: lfoppiano/grobid:0.8.0
     ports:
       - "8070:8070"
       - "8071:8071"

docs/conf.py

@@ -81,7 +81,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:

pyproject.toml

@@ -10,85 +10,113 @@ python = ">= 3.9, <3.10"
 
 invenio-oaiharvester = {git = "https://github.com/inveniosoftware/invenio-oaiharvester.git", tag = "v1.0.0a4"}
 
-invenio-logging = { version = ">=1.3.0,<1.4.0", extras = ["sentry-sdk", "sentry"] }
-
-# Invenio 3.4 base modules. Same as invenio metadata extras without invenio-search-ui
-invenio-indexer = ">=1.2.0,<1.3.0"
-invenio-jsonschemas = ">=1.1.1,<1.2.0"
-invenio-oaiserver = ">1.4.0,<1.5.0"
-invenio-pidstore = ">=1.2.1,<1.3.0"
-invenio-records-rest = ">=1.8.0,<1.9.0"
-invenio-records-ui= ">=1.2.0,<1.3.0"
-invenio-records = ">=1.4.0,<1.7.0"
-invenio-stats = "^1.0.0a18"
-invenio-records-resources = "*"
-invenio-previewer = ">=1.3.5"
-invenio-userprofiles = "<1.2.5"
-invenio = {version = ">=3.4.0,<3.5.0", extras = ["base", "files", "postgresql", "auth", "elasticsearch7", "docs", "tests"]}
-
-uwsgi = ">=2.0"
-uwsgitop = ">=0.11"
-uwsgi-tools = ">=1.1.1"
-orcid = "*"
-python-slugify = "*"
-python3-saml = ">=1.13.0"
+# Invenio core modules
+invenio-app = ">=1.3.4,<1.4.0"
+invenio-base = ">=1.2.16,<1.3.0"
+invenio-cache = ">=1.1.1,<1.2.0"
+invenio-celery = ">=1.2.5,<1.3.0"
+invenio-config = ">=1.0.3,<1.1.0"
+invenio-i18n = ">=2.0.0,<3.0.0"
+invenio-db = {version = ">=1.1.0,<1.2.0", extras = ["postgresql"]}
+
+# Invenio base bundle
+invenio-admin = ">=1.4.0,<1.5.0"
+invenio-assets = ">=2.0.0,<3.0.0"
+invenio-formatter = ">=2.0.0,<3.0.0"
+invenio-logging = {version = ">=2.0.0,<3.0.0"}
+sentry-sdk = ">=1.0.0" # normaly in invenio-logging = {version = ">=2.0.0,<3.0.0", extras = ["sentry_sdk"]}
+invenio-mail = ">=2.0.0,<3.0.0"
+invenio-rest = ">=1.3.0,<1.4.0"
+invenio-theme = ">=2.5.7,<3.0.0"
+
+# Invenio auth bundle
+invenio-access = ">=2.0.0,<3.0.0"
+invenio-accounts = ">=3.0.0,<4.0.0"
+invenio-oauth2server = ">=2.0.0,<3.0.0"
+invenio-oauthclient = ">=3.0.0,<4.0.0"
+invenio-userprofiles = "^2.3.1"
+
+# Invenio metadata bundle
+invenio-indexer = ">=2.2.0,<3.0.0"
+invenio-jsonschemas = ">=1.1.4,<1.2.0"
+invenio-oaiserver = ">=2.2.0,<2.3.0"
+invenio-pidstore = ">=1.3.0,<1.4.0"
+invenio-records-rest = ">=2.2.0,<2.3.0"
+invenio-records-ui = ">=1.2.0,<1.3.0"
+invenio-records = ">=2.1.0,<2.3.0"
+invenio-search-ui = ">=2.4.0,<3.0.0"
+invenio-records-resources = "^4.18.3"
+
+# Files
+invenio-files-rest = '>=2.0.0,<3.0.0'
+invenio-previewer = '>=2.0.0,<3.0.0'
+invenio-records-files = '>=1.2.1,<1.3.0'
+
+## RERO ILS specific python modules
+PyYAML = ">=5.3.1"
+dateparser = ">=1.1.1"
+isbnlib = ">=3.9.1"
+requests = ">=2.20.0"
+polib = "*"
 xmltodict = "*"
-marshmallow = ">=3.0.0,<4.0.0"
-pycountry = "*"
+redisbeat = ">1.2.5, <1.3.0"
+jsonpickle = ">=1.4.1"
+ciso8601 = "*"
+
+## Additionnal constraints on python modules
 markdown-captions = "*"
-bleach = ">3.11"
-wand = ">=0.6.6,<0.7.0"
-python-dotenv = "*"
-flask-cors = ">3.0.8"
-cryptography = ">38.0.2,<40.0"
-netaddr = "*"
-dcxml = "*"
-lxml = ">=4.9.1,<5.0.0"
-webdavclient3 = ">=3.14.5"
-fuzzywuzzy = ">=0.18.0"
-python-Levenshtein = ">=0.12.0"
-polib = ">=1.1.0"
-idutils = ">=1.1.8"
-pillow = ">=9.0.0"
-celery = ">=5.0.0"
-wtforms = "<3.0.0"
-flask-wtf = "<1.0.0"
 zipp = "*"
-pycparser = "*"
-sqlalchemy = "<1.4.0"
-sqlalchemy_continuum = "<2.0.0"
-psycopg2-binary = "<3.0.0"
-MarkupSafe = "<2.1.0"
-jedi = "<0.18.0"
-ipython_genutils = "^0.2.0"
-Flask = "<2.0.0"
-SQLAlchemy = "<1.4.0"
-flask-wiki = "^0.2.2"
-sentry-sdk = "<1.6.1"
-pytest-invenio = ">=1.4.0,<1.4.12"
-dparse = ">=0.5.2"
+
+## Deployment
+python-dotenv = ">=0.13.0"
+pydocstyle = ">=6.1.1"
+
+## Third party optional modules used by RERO ILS
+freezegun = "^1.1.0"
+lazyreader = ">1.0.0"
+jinja2 = ">2.11.2"
+jsonmerge = "^1.8.0"
+num2words = "^0.5.10"
+iso639 = "^0.1.4"
+dcxml = "^0.1.2"
+DeepDiff = "^5.5.0"
+docutils = "<0.18.0"
+wtforms = "<3.0.0"
+poethepoet = "^0.12.3"
 Mako = ">=1.2.2"
-jsonref = "<1.0.0"
-jsonresolver = "<0.3.2"
-setuptools = "<58"
-charset-normalizer = "<2.1.0"
-python-levenshtein = "<0.20.0"
-jsonschema = "<4.0.0"
-pydocstyle = ">=6.1.1,<6.2"
-requests-mock = "^1.11.0"
+rero-invenio-base = "^0.3.0"
+jsonresolver = "*"
+pyparsing = "^3.1.1"
+flask-wiki = "^0.3.1"
+invenio-stats = "^4.0.1"
+fuzzywuzzy = "^0.18.0"
+pycountry = "^23.12.11"
+netaddr = "^0.9.0"
+wand = "^0.6.13"
+webdavclient3 = "^3.14.6"
 pysftp = "^0.2.9"
-rero-invenio-base = "^0.2.1"
+python-slugify = "*"
+orcid = "^1.0.3"
+python3-saml = "^1.16.0"
+python-levenshtein = "^0.23.0"
+jsonschema = "<=4.20.0"
+
+# Production
+uwsgitop = "^0.11"
+uwsgi = "^2.0.23"
+uwsgi-tools = "^1.1.1"
 
 [tool.poetry.dev-dependencies]
+pytest-invenio = ">=2.1.6,<3.0.0"
+Sphinx = ">=4.5.0"
 Flask-Debugtoolbar = ">=0.10.1"
-Sphinx = ">=3.0.0,<4"
-mock = ">=2.0.0"
-pytest-invenio = ">=1.4.1,<1.5.0"
+## RERO ILS specific python packages
 safety = ">=1.8"
+mock = ">=2.0.0"
+autoflake = ">=1.3.1"
 appnope = { version = "*", optional = true }
-autoflake = ">=1.4"
 
-[project.console_scripts]
+[tool.poetry.plugins."console_scripts"]
 sonar = "invenio_app.cli:cli"
 
 [tool.poetry.plugins."flask.commands"]
@@ -215,6 +243,10 @@ stats = "sonar.modules.stats.admin:stats_adminview"
 [tool.poetry.plugins."babel.extractors"]
 json = "sonar.modules.babel_extractors:extract_json"
 
+
+[tool.poetry.group.dev.dependencies]
+requests-mock = "^1.11.0"
+
 [tool.poe.tasks]
 bootstrap = {cmd = "./scripts/bootstrap", help = "Runs bootstrap"}
 console = {cmd = "./scripts/console", help = "Opens invenio shell"}

scripts/bootstrap

@@ -53,21 +53,9 @@ done
 
 title "Bootstrap script"
 
-# Update pip version
-section "Upgrading pip..." "info"
-poetry run pip install "pip<=23.0.0"
-
-# Install setuptools <58 needed to install fs 0.5.4
-section "Install setuptools<58" "info"
-poetry run pip install "setuptools<58"
-
 section "Install poethepoet"
 poetry run pip install poethepoet
 
-# Needed for poetry > 1.1
-section "Install fs"
-poetry run pip install "fs<2.0.0"
-
 if ! $deploy ; then
   if $ci ; then
     section "Updating dependencies" "info"

scripts/functions

@@ -61,3 +61,12 @@ function section {
     color $2
     echo -e "$(tput setaf 0; tput setab $color)\n $1 $(tput sgr0)"
 }
+
+token_create() {
+  if [ -z ${3} ]
+  then
+      invenio utils token-create -n ${1} -u ${2}
+  else
+      invenio utils token-create -n ${1} -u ${2} -t ${3}
+  fi
+}

scripts/server

@@ -58,9 +58,6 @@ script_path=$(dirname "$0")
 if [[ -z "${FLASK_DEBUG}" ]]; then
   export FLASK_DEBUG=True
 fi
-if [[ -z "${FLASK_ENV}" ]]; then
-  export FLASK_ENV="development"
-fi
 
 # Start Worker and Beat
 if $worker; then

scripts/setup

@@ -111,6 +111,13 @@ invenio access allow admin-access role moderator
 invenio access allow admin-access role submitter
 message "Done" "success"
 
+section "Create user for monitoring" "info"
+invenio users create --active --confirm --password monitor monitoring@rero.ch
+invenio roles create monitoring
+invenio roles add monitoring@rero.ch monitoring
+# create token access for monitoring
+token_create monitoring monitoring@rero.ch ${INVENIO_RERO_ACCESS_TOKEN_MONITORING}
+
 # Create a default location for depositing files
 section "Create location for storing files" "info"
 invenio fixtures deposits create
@@ -132,4 +139,8 @@ if $import; then
     invenio oaiharvester harvest -n rerodoc -q -a max=${size}
 fi
 
+section "Initialize wiki search"
+invenio flask_wiki init-index
+invenio flask_wiki index
+
 message "\nInstallation finished successfully" "success"

scripts/test

@@ -29,35 +29,26 @@ if [[ -z "${VIRTUAL_ENV}" ]]; then
   exit 1
 fi
 
-# TODO: Exceptions on safety check
-# +============================+===========+==========================+==========+
-# | package                    | installed | affected                 | ID       |
-# +============================+===========+==========================+==========+
-# | wtforms                    | 2.3.3     | <3.0.0a1                 | 42852    |
-# | werkzeug                   | 1.0.1     | <2.2.3                   | 53325    |
-# | werkzeug                   | 1.0.1     | <2.2.3                   | 53326    |
-# | werkzeug                   | 1.0.1     | >=0,<2.1.1               | 54456    |
-# | sqlalchemy                 | 1.3.24    | <2.0.0b1                 | 51668    |
-# | sqlalchemy-utils           | 0.35.0    | >=0.27.0                 | 42194    |
-# | setuptools                 | 57.5.0    | <65.5.1                  | 52495    |
-# | sentry-sdk                 | 1.6.0     | <1.14.0                  | 53812    |
-# | safety                     | 1.10.3    | <2.2.0                   | 51358    |
-# | py                         | 1.11.0    | <=1.11.0                 | 51457    |
-# | nbconvert                  | 6.4.5     | <6.5.1                   | 50792    |
-# | flask-security             | 3.0.0     | <3.1.0                   | 45183    |
-# | flask-security             | 3.0.0     | >0                       | 44501    |
-# | click                      | 7.1.2     | <8.0.0                   | 47833    |
-# | celery                     | 5.1.2     | <5.2.0                   | 42498    |
-# | celery                     | 5.1.2     | <5.2.2                   | 43738    |
-# | cryptography               | 39.0.2    | <41.0.0                  | 59062    |
-# | cryptography               | 39.0.2    | <41.0.2                  | 59473    |
-# | cryptography               | 39.0.2    | >=0.8, <41.0.3           | 60224    |
-# | cryptography               | 39.0.2    | >=0.8, <41.0.3           | 60225    |
-# | cryptography               | 39.0.2    | >=0.8,<41.0.3            | 60223    |
-# | certifi                    | 2022.12.7 | >=2015.04.28,<2023.07.22 | 59956    |
-# | pillow                     | 9.5.0     | <10.0.1                  | 61489    |
-# +==============================================================================+
-safety check -i 45183 -i 44501 -i 51668 -i 42194 -i 42852 -i 53325 -i 53326 -i 54456 -i 42498 -i 43738 -i 47833 -i 51457 -i 51358 -i 53812 -i 52495 -i 50792 -i 55261 -i 59062 -i 59473 -i 59956 -i 60223 -i 60224 -i 60225 -i 61489 -i 62019 -i 40459 -i 62451 -i 62452 -i 62556 -i 63073
+# -> Vulnerability found in flask-caching version 2.0.1
+#    Vulnerability ID: 40459
+# -> Vulnerability found in sqlalchemy version 1.4.50
+#    Vulnerability ID: 51668
+# -> Vulnerability found in sqlalchemy-utils version 0.38.3
+#    Vulnerability ID: 42194
+# -> Vulnerability found in wtforms version 2.3.3
+#    Vulnerability ID: 42852
+# -> Vulnerability found in werkzeug version 2.2.3
+#    Vulnerability ID: 62019
+# -> Vulnerability found in py version 1.11.0
+#    Vulnerability ID: 51457
+echo "Check vulnerabilities:"
+safety_exceptions="-i 40459 -i 51668 -i 42194 -i 42852 -i 62019 -i 51457"
+msg=$(safety check -o text ${safety_exceptions}) || {
+  echo "Safety vulnerabilites found for packages:" $(safety check -o bare ${safety_exceptions})
+  echo "Run:" "safety check -o screen ${safety_exceptions} | grep -i vulnerability" "for more details"
+  exit 1
+}
+
 pydocstyle sonar tests docs
 isort --check-only --diff "${SCRIPT_PATH}/.."
 autoflake -c -r --remove-all-unused-imports --ignore-init-module-imports . &> /dev/null || {