dependencies: fix security issues

* Bumps dependencies.
* Better verbose prints.
* Cleans code from warnings.

Co-Authored-by: Peter Weber <peter.weber@rero.ch>

.github/workflows/continuous-integration-test.yml

@@ -16,9 +16,9 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Setup node
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
-        node-version: '14'
+        node-version: '16'
 
     - name: Docker compose up
       run: docker-compose up -d

classes.md

@@ -19,8 +19,8 @@ class ReroMefRecord{
     dbcommit(self, reindex=False, forceindex=False)
     reindex(self, forceindex=False)
     get_record_by_pid(cls, pid, with_deleted=False)
-    get_pid_by_id(cls, id)
-    get_persistent_identifier(cls, id)
+    get_pid_by_id(cls, id_)
+    get_persistent_identifier(cls, id_)
     _get_all(cls, with_deleted=False)
     get_all_pids(cls, with_deleted=False, limit=100000)
     get_all_ids(cls, with_deleted=False, limit=100000)

pyproject.toml

@@ -21,12 +21,7 @@ python = ">= 3.9, <3.10"
 ## foo = ">3.0" # finally will upgrade to 5.0
 ## when the python code will be executed a conflict exception will be raised.
 #------------------------------------------------------------------------------
-# Default from Invenio
-lxml = ">=4.9.1"
-marshmallow = ">=3.0.0,<4.0.0"
-uwsgi = ">=2.0"
-uwsgi-tools = ">=1.1.1"
-uwsgitop = ">=0.11"
+
 ## Third party inven## Third party invenio modules used by RERO EBOOKS
 invenio-oaiharvester = { git = "https://github.com/inveniosoftware/invenio-oaiharvester.git", tag = "v1.0.0a4" }
 invenio-search = {version = ">=2.1.0,<3.0.0", extras = ["elasticsearch7"]}
@@ -38,32 +33,41 @@ invenio-cache = ">=1.1.1,<1.2.0"
 invenio-celery = ">=1.2.4,<1.3.0"
 invenio-config = ">=1.0.3,<1.1.0"
 invenio-i18n = ">=2.0.0,<3.0.0"
-invenio-db = {version = ">=1.0.14,<1.1.0", extras = ["postgresql"]}
+invenio-db = {version = ">=1.1.0,<1.2.0", extras = ["postgresql"]}
 # Invenio base bundle
 invenio-admin = ">=1.4.0,<1.5.0"
 invenio-assets = ">=2.0.0,<3.0.0"
-invenio-formatter = ">=1.2.0,<1.3.0"
-invenio-logging = {version = ">=1.3.2,<1.4.0", extras = ["sentry-sdk"]}
-invenio-mail = ">=1.0.2,<1.1.0"
-invenio-rest = ">=1.2.8,<1.3.0"
+invenio-formatter = ">=2.0.0,<3.0.0"
+invenio-logging = {version = ">=2.0.0,<3.0.0"}
+invenio-mail = ">=2.0.0,<3.0.0"
+invenio-rest = ">=1.3.0,<1.4.0"
 invenio-theme = ">=2.0.0,<3.0.0"
 # Invenio auth bundle
-invenio-access = ">=1.4.4,<1.5.0"
-invenio-accounts = ">=2.1.0,<2.2.0"
-invenio-oauth2server = ">=2.0.0,<2.1.0"
-invenio-oauthclient = ">=2.2.0,<3.0.0"
-invenio-userprofiles = ">=2.2.0,<2.3.0"
+invenio-access = ">=2.0.0,<3.0.0"
+invenio-accounts = ">=3.0.0,<4.0.0"
+invenio-oauth2server = ">=2.0.0,<3.0.0"
+invenio-oauthclient = ">=3.0.0,<4.0.0"
+invenio-userprofiles = ">=2.3.0,<3.0.0"
 # Invenio metadata bundle
-invenio-indexer = ">=2.1.0,<2.2.0"
+invenio-indexer = ">=2.2.0,<3.0.0"
 invenio-jsonschemas = ">=1.1.4,<1.2.0"
 invenio-oaiserver = ">=2.2.0,<2.3.0"
 invenio-pidstore = ">=1.3.0,<1.4.0"
 invenio-records-rest = ">=2.2.0,<2.3.0"
 invenio-records-ui = ">=1.2.0,<1.3.0"
-invenio-records = "2.1.0,<2.2.0"
+invenio-records = ">=2.1.0,<2.2.0"
 #invenio-search-ui = ">=2.4.0,<3.0.0"
 # Pinned due to before_first_request deprecation https://flask.palletsprojects.com/en/2.2.x/api/#flask.Flask.before_first_request
+
+# Default from Invenio
+lxml = ">=4.9.1"
+marshmallow = ">=3.0.0,<4.0.0"
+uwsgi = ">=2.0"
+uwsgi-tools = ">=1.1.1"
+uwsgitop = ">=0.11"
+# Pinned due to before_first_request deprecation https://flask.palletsprojects.com/en/2.2.x/api/#flask.Flask.before_first_request
 Flask = ">=2.2.0,<2.3.0"
+sentry-sdk = ">=1.0.0" # normaly in invenio-logging = {version = ">=2.0.0,<3.0.0", extras = ["sentry_sdk"]}
 dojson = ">=1.4.0"
 # TODO: dojson problem = AttributeError: 'Group' object has no attribute 'resultcallback'
 click = "<8.1.0"
@@ -80,7 +84,7 @@ pydocstyle = ">=6.1.1"
 ## RERO ILS specific python modules
 poethepoet = "*"
 # TODO: solve requires invenio-indexer (<2.0.0)
-rero-invenio-base = ">=0.2.1"
+rero-invenio-base = ">=0.2.0"
 sqlitedict = "^2.1.0"
 werkzeug = "<2.3.0"
 sqlalchemy-continuum = ">=1.3.12,<1.4.0"

rero_mef/agents/cli.py

@@ -55,7 +55,7 @@ def create_from_viaf(enqueue, online, verbose, online_verbose,
     """Create MEF and agents from viaf."""
     def get_pids_from_json(json_file):
         """Get all pids from JSON file."""
-        for record in read_json_record(viaf_file):
+        for record in read_json_record(json_file):
             yield record['pid']
 
     click.secho(
@@ -69,26 +69,28 @@ def get_pids_from_json(json_file):
 
     if missing:
         missing_pids, non_existing_pids = AgentMefRecord. \
-            get_all_missing_viaf_pids(
-                verbose=progress or verbose
-            )
+            get_all_missing_viaf_pids(verbose=(progress or verbose))
         progress_bar = progressbar(
             items=missing_pids,
             length=len(missing_pids),
-            verbose=progress
+            verbose=progress,
+            label='VIAF missing'
         )
     elif viaf_file:
         progress_bar = progressbar(
             items=get_pids_from_json(viaf_file),
             length=number_records_in_file(viaf_file.name, 'json'),
-            verbose=progress
+            verbose=progress,
+            label='VIAF file'
         )
     else:
         progress_bar = progressbar(
             items=AgentViafRecord.get_all_pids(),
             length=counts['viaf']['old'],
-            verbose=progress
+            verbose=progress,
+            label='VIAF all'
         )
+    click.echo('Create MEF and agents from VIAF')
     for pid in progress_bar:
         if enqueue:
             task = task_create_mef_and_agents_from_viaf.delay(
@@ -111,7 +113,7 @@ def get_pids_from_json(json_file):
     if non_existing_pids:
         click.echo(
             f'Clean VIAF pids from MEF records: {len(non_existing_pids)}')
-        for pid, viaf_pid in non_existing_pids.items():
+        for pid, _ in non_existing_pids.items():
             # TODO: clean MEF records with non existing VIAF pids:
             pass
 
@@ -185,7 +187,7 @@ def create_csv_mef(viaf_metadata_file, output_directory, verbose):
     click.secho(
         f'  VIAF input file: {viaf_metadata_file}',
         err=True)
-    message = f'  CSV output files: {pidstore}, {metadata}'
+    # message = f'  CSV output files: {pidstore}, {metadata}'
 
     count = create_mef_files(
             viaf_metadata_file_name=viaf_metadata_file,

rero_mef/agents/gnd/tasks.py

@@ -109,7 +109,7 @@ def gnd_get_record(id_, debug=False):
             if records := parse_xml_to_array(BytesIO(response.content)):
                 trans_record = Transformation(records[0]).json
                 pid = trans_record.get('pid')
-                if id_ != trans_record.get('pid'):
+                if id_ != pid:
                     msg = f'{msg} | PID changed: {id_} -> {pid}'
                     trans_record = None
                 else:
@@ -119,6 +119,7 @@ def gnd_get_record(id_, debug=False):
         else:
             msg = f'{msg} | HTTP Error: {status_code}'
     except Exception as err:
+        trans_record = None
         msg = f'{msg} | Error: {err}'
         if debug:
             raise

rero_mef/agents/mef/api.py

@@ -83,7 +83,8 @@ def get_all_missing_viaf_pids(cls, verbose=False):
         progress = progressbar(
             items=AgentViafRecord.get_all_pids(),
             length=AgentViafRecord.count(),
-            verbose=verbose
+            verbose=verbose,
+            label='VIAF all'
         )
         missing_pids = {pid: 1 for pid in progress}
         if verbose:
@@ -92,7 +93,8 @@ def get_all_missing_viaf_pids(cls, verbose=False):
         progress = progressbar(
             items=query.source(['pid', 'viaf_pid']).scan(),
             length=query.count(),
-            verbose=True
+            verbose=verbose,
+            label='VIAF from MEF'
         )
         non_existing_pids = {hit.pid: hit.viaf_pid for hit in progress
                              if not missing_pids.pop(hit.viaf_pid, None)}

rero_mef/agents/viaf/api.py

@@ -32,8 +32,6 @@
 from .. import AgentGndRecord, AgentIdrefRecord, AgentMefRecord, \
     AgentReroRecord
 from ..api import Action, ReroIndexer, ReroMefRecord
-from ..mef.api import AgentMefRecord
-from ..utils import get_entity_class
 from ...filter import exists_filter
 from ...utils import add_md5, get_entity_class, progressbar, \
     requests_retry_session
@@ -209,7 +207,7 @@ def update_online(agent_class, pid, online):
             if agent_class.provider.pid_type in online:
                 data, msg = agent_class.get_online_record(id_=pid)
                 if online_verbose:
-                    click.echo(msg)
+                    click.echo(f'\n{msg}')
                 if data and not data.get('NO TRANSFORMATION'):
                     agent_record, action = agent_class.create_or_update(
                         data=data, dbcommit=dbcommit, reindex=reindex)
@@ -320,22 +318,22 @@ def reindex(self, forceindex=False):
         return result
 
     @classmethod
-    def get_online_record(cls, viaf_source_code, pid, format=None):
+    def get_online_record(cls, viaf_source_code, pid, rec_format=None):
         """Get VIAF record.
 
         Get's the VIAF record from:
         http://www.viaf.org/viaf/sourceID/{source_code}|{pid}
 
         :param viaf_source_code: agent source code
         :param pid: pid for agent source code
-        :param format: raw = get the not transformed VIAF record
+        :param rec_format: raw = get the not transformed VIAF record
                        link = get the VIAF link record
         :returns: VIAF record as json
         """
         viaf_format = '/viaf.json'
-        if format == 'link':
+        if rec_format == 'link':
             viaf_format = '/justlinks.json'
-            format = 'raw'
+            rec_format = 'raw'
         viaf_url = current_app.config.get('RERO_MEF_VIAF_BASE_URL')
         url = f'{viaf_url}/viaf'
         if viaf_source_code.upper() == 'VIAF':
@@ -346,7 +344,7 @@ def get_online_record(cls, viaf_source_code, pid, format=None):
         result = {}
         if response.status_code == requests.codes.ok:
             msg = f'VIAF get: {pid:<15} {url} | OK'
-            if format == 'raw':
+            if rec_format == 'raw':
                 return response.json(), msg
             data_json = response.json()
             result['pid'] = data_json.get('viafID')
@@ -394,7 +392,6 @@ def update_online(self, dbcommit=False, reindex=False):
         :param reindex: Reindex record.
         :returns: record and actions message.
         """
-        from rero_mef.api import Action
         online_data, _ = self.get_online_record(
             viaf_source_code='VIAF',
             pid=self.pid
@@ -491,7 +488,7 @@ def delete(self, force=True, dbcommit=False, delindex=False):
             AgentMefRecord.flush_indexes()
         # recreate MEF records for agents
         for agent_record in old_agent_records.values():
-            mef, test = agent_record.create_or_update_mef(
+            mef, _ = agent_record.create_or_update_mef(
                 dbcommit=True,
                 reindex=True
             )

rero_mef/api.py

@@ -236,13 +236,12 @@ def replace(self, data, commit=False, dbcommit=False, reindex=False):
         if self.get('md5'):
             new_data = add_md5(new_data)
         self.clear()
-        self = self.update(
+        return self.update(
             data=new_data,
             commit=commit,
             dbcommit=dbcommit,
             reindex=reindex
         )
-        return self
 
     def dbcommit(self, reindex=False, forceindex=False):
         """Commit changes to db."""
@@ -285,18 +284,18 @@ def get_record_by_pid(cls, pid, with_deleted=False):
                 db.session.rollback()
 
     @classmethod
-    def get_pid_by_id(cls, id):
+    def get_pid_by_id(cls, id_):
         """Get pid by uuid."""
-        persistent_identifier = cls.get_persistent_identifier(id)
+        persistent_identifier = cls.get_persistent_identifier(id_)
         return str(persistent_identifier.pid_value)
 
     @classmethod
-    def get_persistent_identifier(cls, id):
+    def get_persistent_identifier(cls, id_):
         """Get Persistent Identifier."""
         return PersistentIdentifier.get_by_object(
             cls.provider.pid_type,
             cls.object_type,
-            id
+            id_
         )
 
     @classmethod
@@ -374,8 +373,8 @@ def get_all_ids(cls, with_deleted=False, limit=100000, date=None):
     @classmethod
     def get_all_records(cls, with_deleted=False, limit=100000):
         """Get all records. Return a generator iterator."""
-        for id in cls.get_all_ids(with_deleted=with_deleted, limit=limit):
-            yield cls.get_record(id)
+        for id_ in cls.get_all_ids(with_deleted=with_deleted, limit=limit):
+            yield cls.get_record(id_)
 
     @classmethod
     def count(cls, with_deleted=False):