Merge branch 'develop' into release-2.2

.travis.yml

@@ -31,22 +31,27 @@ jobs:
     - name: Aether CORE tests (Extractor, Kernel, Producer)
       stage: test
       env: 'TEST_MODE=core'
+      if: type = pull_request
 
     - name: Aether modules tests (ODK)
       stage: test
       env: 'TEST_MODE=modules'
+      if: type = pull_request
 
     - name: Aether UI tests (UI Module)
       stage: test
       env: 'TEST_MODE=ui'
+      if: type = pull_request
 
     - name: Integration tests (Kernel with Kafka/Zookeeper and Client)
       stage: test
       env: 'TEST_MODE=integration'
+      if: type = pull_request
 
     - name: Test deployment process
       stage: test
       script: ./scripts/test_deployment.sh
+      if: type = pull_request
 
     - name: Publish images in Docker Hub
       stage: deploy

aether-client-library/aether/client/__init__.py

@@ -153,7 +153,7 @@ def show_request(operation, *args, **kwargs):
     request_options = kwargs.pop('_request_options', {})
     request_params = construct_request(
         operation, request_options, **kwargs)
-    return([kwargs, request_params])
+    return [kwargs, request_params]
 
 
 '''
@@ -176,7 +176,7 @@ def __init__(self, resource, also_return_response=True, swagger_spec=None):
         self.name = resource.name
         # The only way to be able to form coherent exceptions is to catch these
         # common types and wrap them in our own, exposing the status and error
-        # feeback from the API.
+        # feedback from the API.
         self.handled_exceptions = [
             bravado.exception.HTTPBadRequest,
             bravado.exception.HTTPBadGateway,

aether-client-library/aether/client/test/test.py

@@ -22,42 +22,42 @@
 
 
 def test_1_check_fixture_creation(client, project, schemas, schemadecorators, mapping):
-    assert(project.id is not None)
+    assert (project.id is not None)
     client_schemas = list(client.schemas.paginated('list'))
     assert len(schemas) != 0
-    assert(len(client_schemas) == len(schemas))
+    assert (len(client_schemas) == len(schemas))
     client_sd = list(client.schemadecorators.paginated('list'))
-    assert(len(client_sd) == len(schemas))
-    assert(mapping.id is not None)
+    assert (len(client_sd) == len(schemas))
+    assert (mapping.id is not None)
 
 
 def test_2_count_schemas(client, schemas):
     ct = client.schemas.count('list')
-    assert(ct == len(schemas))
+    assert (ct == len(schemas))
 
 
 def test_3_first_schema(client, schemas):
     first = client.schemas.first('list', ordering='modified')
-    assert(first.id == schemas[0].id)
+    assert (first.id == schemas[0].id)
 
 
 def test_4_iterate_schemas(client, schemas):
     _schemas = list(client.schemas.paginated('list'))
-    assert(len(_schemas) == len(schemas))
+    assert (len(_schemas) == len(schemas))
 
 
 def test_5_make_entities(client, single_entities, bulk_entities):
     single = single_entities(1)
-    assert(single is not None)
+    assert (single is not None)
     many = bulk_entities(10)
-    assert(many is not None)
+    assert (many is not None)
     entities = client.entities.paginated('list')
     i = 0
     for e in entities:
         if LOG_LEVEL == 'DEBUG':
             print(e)
         i += 1
-    assert(i == 11)
+    assert (i == 11)
 
 
 # After this point, the artifacts we cached are invalidated.
@@ -67,38 +67,38 @@ def test_6_update_project(client):
     project.name = new_name
     client.projects.update(id=project.id, data=project)
     project = client.projects.first('list')
-    assert(project.name == new_name)
+    assert (project.name == new_name)
 
 
 def test_7_update_project_partial(client, project):
     new_name = 'yet another new name'
     pkg = {'name': new_name}
     client.projects.partial_update(id=project.id, data=pkg)
     retrieved = client.projects.first('list')
-    assert(retrieved.name == new_name)
+    assert (retrieved.name == new_name)
 
 
 def test_7_delete_project(client, project):
     client.projects.delete(id=project.id)
     projects = list(client.projects.paginated('list'))
-    assert(len(projects) == 0)
+    assert (len(projects) == 0)
 
 
 def test_8_check_bad_url():
     try:
         c = Client('http://localhost/bad-url', 'user', 'pw', auth_type='basic')
         c.get('projects')
     except bravado.exception.BravadoConnectionError:
-        assert(True)
+        assert (True)
     else:
-        assert(False)
+        assert (False)
 
 
 def test_9_check_bad_credentials():
     try:
         c = Client(URL, 'user', 'pw', realm='dev', auth_type='basic')
         c.get('projects')
     except bravado.exception.HTTPForbidden:
-        assert(True)
+        assert (True)
     else:
-        assert(False)
+        assert (False)

aether-client-library/conf/pip/requirements.txt

@@ -10,7 +10,7 @@ requests[security]
 requests_oauthlib
 
 # Test libraries
-flake8
+flake8<6
 flake8-quotes
 pytest
 pytest-runner

aether-entity-extraction-module/aether/extractor/tests/test_manager.py

@@ -75,7 +75,7 @@ def load_redis(redis):
         build_redis_key(ARTEFACT_NAMES.mappingsets, TENANT, MAPPINGSET['id']),
         json.dumps(MAPPINGSET)
     )
-    assert(
+    assert (
         get_from_redis_or_kernel(
             id=MAPPINGSET['id'],
             model_type=ARTEFACT_NAMES.mappingsets,
@@ -89,7 +89,7 @@ def load_redis(redis):
             build_redis_key(ARTEFACT_NAMES.mappings, TENANT, m['id']),
             json.dumps(m)
         )
-        assert(
+        assert (
             get_from_redis_or_kernel(
                 id=m['id'],
                 model_type=ARTEFACT_NAMES.mappings,
@@ -103,7 +103,7 @@ def load_redis(redis):
             build_redis_key(ARTEFACT_NAMES.schemas, TENANT, s['id']),
             json.dumps(s)
         )
-        assert(
+        assert (
             get_from_redis_or_kernel(
                 id=s['id'],
                 model_type=ARTEFACT_NAMES.schemas,
@@ -117,7 +117,7 @@ def load_redis(redis):
             build_redis_key(ARTEFACT_NAMES.schemadecorators, TENANT, sd['id']),
             json.dumps(sd)
         )
-        assert(
+        assert (
             get_from_redis_or_kernel(
                 id=sd['id'],
                 model_type=ARTEFACT_NAMES.schemadecorators,

aether-entity-extraction-module/conf/pip/primary-requirements.txt

@@ -18,6 +18,7 @@ gevent
 # Test libraries
 coverage
 fakeredis
+flake8<6
 flake8-quotes
 pytest
 tblib

aether-entity-extraction-module/conf/pip/requirements.txt

@@ -14,43 +14,41 @@
 
 aether.python==1.3.0
 async-timeout==4.0.2
-attrs==21.4.0
+attrs==22.2.0
 avro-python3==1.10.2
-certifi==2022.6.15
-charset-normalizer==2.0.12
-coverage==6.4.1
+certifi==2022.12.7
+charset-normalizer==3.0.1
+coverage==7.0.5
 decorator==5.1.1
-Deprecated==1.2.13
 eha-jsonpath==0.6.0
-fakeredis==1.8.1
-flake8==4.0.1
-flake8-quotes==3.3.1
-gevent==21.12.0
-greenlet==1.1.2
-idna==3.3
-importlib-resources==5.8.0
-iniconfig==1.1.1
+exceptiongroup==1.1.0
+fakeredis==2.4.0
+flake8==5.0.4
+flake8-quotes==3.3.2
+gevent==22.10.2
+greenlet==2.0.1
+idna==3.4
+importlib-resources==5.10.2
+iniconfig==2.0.0
 jsonpath-ng==1.5.3
-jsonschema==4.6.0
-mccabe==0.6.1
-packaging==21.3
+jsonschema==4.17.3
+mccabe==0.7.0
+packaging==23.0
+pkgutil_resolve_name==1.3.10
 pluggy==1.0.0
 ply==3.11
-py==1.11.0
-pycodestyle==2.8.0
-pyflakes==2.4.0
-pyparsing==3.0.9
-pyrsistent==0.18.1
-pytest==7.1.2
-redis==4.3.3
-requests==2.28.0
+pycodestyle==2.9.1
+pyflakes==2.5.0
+pyrsistent==0.19.3
+pytest==7.2.1
+redis==4.4.2
+requests==2.28.2
 six==1.16.0
 sortedcontainers==2.4.0
 spavro==1.1.24
 tblib==1.7.0
 tomli==2.0.1
-urllib3==1.26.9
-wrapt==1.14.1
-zipp==3.8.0
-zope.event==4.5.0
-zope.interface==5.4.0
+urllib3==1.26.14
+zipp==3.11.0
+zope.event==4.6
+zope.interface==5.5.2

aether-kernel/aether/kernel/api/serializers.py

@@ -257,8 +257,9 @@ class EntityListSerializer(serializers.ListSerializer):
 
     def create(self, validated_data):
         if (bulk_size := len(validated_data)) > settings.MAX_BULK_RECORDS:
-            raise(serializers.ValidationError(
-                f'{bulk_size} exceeds max: {settings.MAX_BULK_RECORDS} for a single request.'))
+            raise serializers.ValidationError(
+                f'{bulk_size} exceeds max: {settings.MAX_BULK_RECORDS} for a single request.'
+            )
 
         entities = []
         # remove helper field and validate entity
@@ -270,7 +271,7 @@ def create(self, validated_data):
                 entity.clean()
                 entities.append(entity)
             except Exception as e:
-                raise(serializers.ValidationError(str(e)))
+                raise serializers.ValidationError(str(e))
 
         # bulk database operation
         created_entities = models.Entity.objects.bulk_create(entities, ignore_conflicts=True)
@@ -392,8 +393,9 @@ class SubmissionListSerializer(serializers.ListSerializer):
 
     def create(self, validated_data):
         if (bulk_size := len(validated_data)) > settings.MAX_BULK_RECORDS:
-            raise(serializers.ValidationError(
-                f'{bulk_size} exceeds max: {settings.MAX_BULK_RECORDS} for a single request.'))
+            raise serializers.ValidationError(
+                f'{bulk_size} exceeds max: {settings.MAX_BULK_RECORDS} for a single request.'
+            )
 
         for s in validated_data:
             if not s.get('mappingset'):

aether-kernel/conf/pip/primary-requirements.txt

@@ -17,7 +17,6 @@
 
 # Aether Django SDK library with extras
 aether.sdk[cache,server,storage,test]
-django<4
 
 
 # Aether Python library