Merge 0e98ba9 into feda000

.github/workflows/continuous-integration-test.yml

@@ -7,8 +7,11 @@ env:
 jobs:
   test:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        dependencies: ['dev', 'deploy']
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Install Libraries
       run: |
@@ -22,9 +25,9 @@ jobs:
         sudo sed -i 's/<policy domain="coder" rights="none" pattern="PDF" \/>/<policy domain="coder" rights="read" pattern="PDF" \/>/g' /etc/ImageMagick-6/policy.xml
 
     - name: Setup node
-      uses: actions/setup-node@v2
+      uses: actions/setup-node@v3
       with:
-        node-version: '12'
+        node-version: '14'
 
     - name: Docker compose up
       run: docker-compose up -d
@@ -42,6 +45,17 @@ jobs:
         poetry run ./scripts/bootstrap
         pip install --upgrade coveralls
 
+    - name: Bootstrap
+      if: ${{ matrix.dependencies == 'dev' }}
+      run: |
+        poetry run ./scripts/bootstrap --ci
+        pip install --upgrade coveralls
+
+    - name: Bootstrap deploy
+      if: ${{ matrix.dependencies == 'deploy' }}
+      run: |
+        poetry run ./scripts/bootstrap --ci --deploy E2E=yes
+
     - name: Run Test
       run: poetry run ./scripts/test
 

Dockerfile

@@ -27,6 +27,8 @@
 ARG VERSION=latest
 FROM sonar-base:${VERSION}
 
+USER 0
+
 # Copy files
 COPY ./ ${WORKING_DIR}/src
 WORKDIR ${WORKING_DIR}/src
@@ -35,9 +37,12 @@ COPY ./docker/uwsgi/ ${INVENIO_INSTANCE_PATH}
 # Change owner
 RUN chown -R invenio:invenio ${WORKING_DIR}
 
+USER 1000
+
 # Run bootstrap
 ENV TERM=xterm-256color
 ARG UI_TGZ=""
+ENV INVENIO_COLLECT_STORAGE='flask_collect.storage.file'
 RUN poetry run ./scripts/bootstrap --deploy --ui ${UI_TGZ}
 
 ENTRYPOINT [ "bash", "-c"]

Dockerfile.base

@@ -22,7 +22,7 @@
 # and includes Pip, Pipenv, Node.js, NPM and some few standard libraries
 # Invenio usually needs.
 
-FROM python:3.9-slim-buster
+FROM python:3.9-slim-bullseye
 
 # require debian packages
 RUN apt-get upgrade -y && apt-get update -y
@@ -56,8 +56,4 @@ RUN useradd invenio --uid ${INVENIO_USER_ID}  --home ${WORKING_DIR} && \
     chown -R invenio:invenio ${WORKING_DIR} && \
     chmod -R go+w ${WORKING_DIR}
 
-# Install dependencies
-RUN poetry run pip install --upgrade pip "setuptools<58"
-RUN poetry install --no-dev
-
-
+USER 1000

pyproject.toml

@@ -68,6 +68,12 @@ sentry-sdk = "<1.6.1"
 pytest-invenio = ">=1.4.0,<1.4.12"
 dparse = ">=0.5.2"
 Mako = ">=1.2.2"
+jsonref = "<1.0.0"
+jsonresolver = "<0.3.2"
+setuptools = "<58"
+charset-normalizer = "<2.1.0"
+python-levenshtein = "<0.20.0"
+jsonschema = "<4.0.0"
 
 [tool.poetry.dev-dependencies]
 Flask-Debugtoolbar = ">=0.10.1"

scripts/bootstrap

@@ -21,6 +21,7 @@ source $(dirname "$0")/functions
 set -e
 
 deploy=false
+ci=false
 ui_tgz=""
 
 # Poetry is a mandatory condition to launch this program!
@@ -38,6 +39,9 @@ while test $# -gt 0; do
               shift
             fi
             ;;
+        -c|--ci)
+          # We are probably running webaction, check for env variables
+          ci=true ;;
         --deploy)
             deploy=true
             ;;
@@ -46,6 +50,7 @@ while test $# -gt 0; do
     shift
 done
 
+
 title "Bootstrap script"
 
 # Update pip version
@@ -63,10 +68,19 @@ poetry run pip install poethepoet
 section "Install fs"
 poetry run pip install "fs<2.0.0"
 
-# Installs all packages specified in poetry.lock
-if [ "$deploy" = false ]; then
-  section "Updating dependencies" "info"
+if ! $deploy ; then
+  if $ci ; then
+    section "Updating dependencies" "info"
+    poetry update
+  fi
+fi
+
+if ! $deploy ; then
+  section "Install the application" "info"
   poetry install
+else
+  section "Install the application for production" "info"
+  poetry install --only main
 fi
 
 # Collect static files

scripts/test

@@ -34,7 +34,10 @@ fi
 # | package                    | installed | affected                 | ID       |
 # +============================+===========+==========================+==========+
 # | wtforms                    | 2.3.3     | <3.0.0a1                 | 42852    |
+# | wheel                      | 0.37.1    | <0.38.0                  | 51499    |
 # | sqlalchemy-utils           | 0.35.0    | >=0.27.0                 | 42194    |
+# | safety                     | 1.10.3    | <2.2.0                   | 51358    |
+# | py                         | 1.11.0    | <=1.11.0                 | 51457    |
 # | nbconvert                  | 6.4.5     | <6.5.1                   | 50792    |
 # | lxml                       | 4.7.0     | <4.9.1                   | 50748    |
 # | flask-security             | 3.0.0     | <3.1.0                   | 45183    |
@@ -43,7 +46,7 @@ fi
 # | celery                     | 5.1.2     | <5.2.0                   | 42498    |
 # | celery                     | 5.1.2     | <5.2.2                   | 43738    |
 # +==============================================================================+
-safety check -i 42852 -i 42050 -i 42194 -i 45183 -i 42498 -i 44501 -i 43738 -i 47833 -i 50748 -i 50792
+safety check -i 42852 -i 51499 -i 42194 -i 51358 -i 51457 -i 50792 -i 50748 -i 45183 -i 44501 -i 47833 -i 42498 -i 43738
 pydocstyle sonar tests docs
 isort --check-only --diff "${SCRIPT_PATH}/.."
 autoflake -c -r --remove-all-unused-imports --ignore-init-module-imports . &> /dev/null || {

sonar/modules/documents/serializers/dc.py

@@ -18,6 +18,7 @@
 """Dublin Core serializer."""
 
 from flask_resources.serializers import SerializerMixin
+from invenio_oaiserver.utils import sanitize_unicode
 from lxml import etree
 
 from sonar.modules.documents.serializers.schemas.dc import DublinCoreSchema
@@ -102,7 +103,7 @@ def serialize_dict_to_etree(self, data):
                         f'{{http://purl.org/dc/elements/1.1/}}{elements[key]}',
                         attrs
                         )
-                    field.text = val
+                    field.text = sanitize_unicode(val)
         return root
 
 

tests/ui/documents/test_dc_schema.py

@@ -201,6 +201,19 @@ def test_descriptions_attributes(minimal_document):
             'value': 'Description 2'
         }
     ]
+    minimal_document['abstracts'] = [{
+        'language': 'fre',
+        'value': 'sous\x02évalués'
+    }]
+
+
+def test_descriptions_xml_control_char(minimal_document):
+    minimal_document['abstracts'] = [{
+        'language': 'fre',
+        'value': 'sous\x02évalués'
+    }]
+    assert SonarDublinCoreXMLSerializer().serialize_object_xml(dict(_source=minimal_document))
+
 
 def test_formats(minimal_document):
     result = SonarDublinCoreXMLSerializer().transform_record(minimal_document)