From 5a82dda236e1aa2df101278ce9b2522232b12e36 Mon Sep 17 00:00:00 2001 From: obdulia-losantos Date: Mon, 13 Sep 2021 20:00:51 +0200 Subject: [PATCH 1/4] fix: docker-compose file --- docker-compose-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-base.yml b/docker-compose-base.yml index a4b001e10..fd4615c8d 100644 --- a/docker-compose-base.yml +++ b/docker-compose-base.yml @@ -201,7 +201,7 @@ services: volumes: - ./aether-entity-extraction-module:/code # to speed up Aether Python Library development changes - # - ${AETHER_PYTHON_PATH:-../aether-python-library}/aether/python:/var/run/aether/venv/lib/python3.7/site-packages/aether/python + # - ${AETHER_PYTHON_PATH:-../aether-python-library}/aether/python:/var/run/aether/venv/lib/python3.8/site-packages/aether/python command: start From e4eca8753b4a31f08364e47aeefdd36989ccfdee Mon Sep 17 00:00:00 2001 From: Obdulia Losantos Date: Tue, 5 Oct 2021 16:01:15 +0200 Subject: [PATCH 2/4] chore: upgrade dependencies (#963) * chore: upgrade dependencies * fix: client requirements --- .../conf/pip/requirements.txt | 1 + aether-client-library/setup.py | 1 + .../conf/pip/requirements.txt | 10 ++-- aether-kernel/conf/pip/requirements.txt | 46 +++++++++---------- aether-odk-module/conf/pip/requirements.txt | 38 ++++++++------- aether-producer/conf/pip/requirements.txt | 12 ++--- aether-ui/aether/ui/assets/package.json | 2 +- aether-ui/conf/pip/requirements.txt | 40 ++++++++-------- 8 files changed, 73 insertions(+), 77 deletions(-) diff --git a/aether-client-library/conf/pip/requirements.txt b/aether-client-library/conf/pip/requirements.txt index 5741bdfb8..23dd1e017 100644 --- a/aether-client-library/conf/pip/requirements.txt +++ b/aether-client-library/conf/pip/requirements.txt @@ -5,6 +5,7 @@ ################################################################################ bravado +jsonschema[format]<4 requests[security] requests_oauthlib diff --git a/aether-client-library/setup.py b/aether-client-library/setup.py index f0e6c4336..44e68636e 100644 --- a/aether-client-library/setup.py +++ b/aether-client-library/setup.py @@ -40,6 +40,7 @@ def read(f): install_requires=[ 'bravado', + 'jsonschema[format]<4', 'requests[security]', 'requests_oauthlib' ], diff --git a/aether-entity-extraction-module/conf/pip/requirements.txt b/aether-entity-extraction-module/conf/pip/requirements.txt index 954e80cb4..6db305dd6 100644 --- a/aether-entity-extraction-module/conf/pip/requirements.txt +++ b/aether-entity-extraction-module/conf/pip/requirements.txt @@ -16,19 +16,19 @@ aether.python==1.3.0 attrs==21.2.0 avro-python3==1.10.2 certifi==2021.5.30 -charset-normalizer==2.0.4 -coverage==5.5 +charset-normalizer==2.0.6 +coverage==6.0 decorator==5.1.0 eha-jsonpath==0.6.0 fakeredis==1.6.1 flake8==3.9.2 flake8-quotes==3.3.0 gevent==21.8.0 -greenlet==1.1.1 +greenlet==1.1.2 idna==3.2 iniconfig==1.1.1 jsonpath-ng==1.5.3 -jsonschema==3.2.0 +jsonschema==4.0.1 mccabe==0.6.1 packaging==21.0 pluggy==1.0.0 @@ -46,6 +46,6 @@ sortedcontainers==2.4.0 spavro==1.1.24 tblib==1.7.0 toml==0.10.2 -urllib3==1.26.6 +urllib3==1.26.7 zope.event==4.5.0 zope.interface==5.4.0 diff --git a/aether-kernel/conf/pip/requirements.txt b/aether-kernel/conf/pip/requirements.txt index fbb2bf3bd..14f6acdf7 100644 --- a/aether-kernel/conf/pip/requirements.txt +++ b/aether-kernel/conf/pip/requirements.txt @@ -18,24 +18,23 @@ asgiref==3.4.1 attrs==21.2.0 autopep8==1.5.7 avro-python3==1.10.2 -boto3==1.18.40 -botocore==1.21.40 -cachetools==4.2.2 +boto3==1.18.54 +botocore==1.21.54 +cachetools==4.2.4 certifi==2021.5.30 -cffi==1.14.6 -charset-normalizer==2.0.4 +charset-normalizer==2.0.6 configparser==5.0.2 coreapi==2.3.3 coreschema==0.0.4 -coverage==5.5 +coverage==6.0 decorator==5.1.0 -Django==3.2.7 +Django==3.2.8 django-cacheops==6.0 django-cleanup==5.2.0 -django-cors-headers==3.8.0 +django-cors-headers==3.9.0 django-debug-toolbar==3.2.2 -django-dynamic-fixture==3.1.1 -django-filter==2.4.0 +django-dynamic-fixture==3.1.2 +django-filter==21.1 django-minio-storage==0.3.10 django-model-utils==4.1.1 django-postgrespool2==2.0.1 @@ -53,56 +52,55 @@ flake8==3.9.2 flake8-quotes==3.3.0 funcy==1.16 google-api-core==2.0.1 -google-auth==2.0.2 +google-auth==2.2.1 google-cloud-core==2.0.0 -google-cloud-storage==1.42.1 -google-crc32c==1.1.2 -google-resumable-media==2.0.2 +google-cloud-storage==1.42.3 +google-crc32c==1.2.0 +google-resumable-media==2.0.3 googleapis-common-protos==1.53.0 gprof2dot==2021.2.21 -greenlet==1.1.1 +greenlet==1.1.2 idna==3.2 inflection==0.5.1 itypes==1.2.0 -Jinja2==3.0.1 +Jinja2==3.0.2 jmespath==0.10.0 jsonpath-ng==1.5.3 -jsonschema==3.2.0 +jsonschema==4.0.1 lxml==4.6.3 Markdown==3.3.4 MarkupSafe==2.0.1 mccabe==0.6.1 minio==6.0.2 -openpyxl==3.0.8 +openpyxl==3.0.9 packaging==21.0 ply==3.11 prometheus-client==0.11.0 -protobuf==3.17.3 +protobuf==3.18.0 psycopg2-binary==2.9.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycodestyle==2.7.0 -pycparser==2.20 pyflakes==2.3.1 Pygments==2.10.0 pyparsing==2.4.7 pyrsistent==0.18.0 python-dateutil==2.8.2 python-json-logger==2.0.2 -pytz==2021.1 +pytz==2021.3 redis==3.5.3 requests==2.26.0 rsa==4.7.2 ruamel.yaml==0.17.16 ruamel.yaml.clib==0.2.6 s3transfer==0.5.0 -sentry-sdk==1.3.1 +sentry-sdk==1.4.3 six==1.16.0 spavro==1.1.24 -SQLAlchemy==1.4.23 +SQLAlchemy==1.4.25 sqlparse==0.4.2 tblib==1.7.0 toml==0.10.2 uritemplate==3.0.1 -urllib3==1.26.6 +urllib3==1.26.7 uWSGI==2.0.19.1 diff --git a/aether-odk-module/conf/pip/requirements.txt b/aether-odk-module/conf/pip/requirements.txt index c61a992c1..db47ee615 100644 --- a/aether-odk-module/conf/pip/requirements.txt +++ b/aether-odk-module/conf/pip/requirements.txt @@ -15,18 +15,17 @@ aether.sdk==1.4.0 asgiref==3.4.1 autopep8==1.5.7 -boto3==1.18.40 -botocore==1.21.40 -cachetools==4.2.2 +boto3==1.18.54 +botocore==1.21.54 +cachetools==4.2.4 certifi==2021.5.30 -cffi==1.14.6 -charset-normalizer==2.0.4 +charset-normalizer==2.0.6 configparser==5.0.2 -coverage==5.5 -Django==3.2.7 +coverage==6.0 +Django==3.2.8 django-cacheops==6.0 django-cleanup==5.2.0 -django-cors-headers==3.8.0 +django-cors-headers==3.10.0 django-debug-toolbar==3.2.2 django-minio-storage==0.3.10 django-postgrespool2==2.0.1 @@ -42,16 +41,16 @@ flake8-quotes==3.3.0 FormEncode==1.3.1 funcy==1.16 google-api-core==2.0.1 -google-auth==2.0.2 +google-auth==2.2.1 google-cloud-core==2.0.0 -google-cloud-storage==1.42.1 -google-crc32c==1.1.2 -google-resumable-media==2.0.2 +google-cloud-storage==1.42.3 +google-crc32c==1.2.0 +google-resumable-media==2.0.3 googleapis-common-protos==1.53.0 gprof2dot==2021.2.21 -greenlet==1.1.1 +greenlet==1.1.2 idna==3.2 -Jinja2==3.0.1 +Jinja2==3.0.2 jmespath==0.10.0 linecache2==1.0.0 lxml==4.6.3 @@ -60,32 +59,31 @@ MarkupSafe==2.0.1 mccabe==0.6.1 minio==6.0.2 prometheus-client==0.11.0 -protobuf==3.17.3 +protobuf==3.18.0 psycopg2-binary==2.9.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycodestyle==2.7.0 -pycparser==2.20 pyflakes==2.3.1 Pygments==2.10.0 python-dateutil==2.8.2 python-json-logger==2.0.2 -pytz==2021.1 +pytz==2021.3 pyxform==1.6.0 redis==3.5.3 requests==2.26.0 rsa==4.7.2 s3transfer==0.5.0 -sentry-sdk==1.3.1 +sentry-sdk==1.4.3 six==1.16.0 spavro==1.1.24 -SQLAlchemy==1.4.23 +SQLAlchemy==1.4.25 sqlparse==0.4.2 tblib==1.7.0 toml==0.10.2 traceback2==1.4.0 unicodecsv==0.14.1 unittest2==1.1.0 -urllib3==1.26.6 +urllib3==1.26.7 uWSGI==2.0.19.1 xlrd==1.2.0 diff --git a/aether-producer/conf/pip/requirements.txt b/aether-producer/conf/pip/requirements.txt index 1ad579ff9..8412c73a7 100644 --- a/aether-producer/conf/pip/requirements.txt +++ b/aether-producer/conf/pip/requirements.txt @@ -14,18 +14,18 @@ attrs==21.2.0 certifi==2021.5.30 -charset-normalizer==2.0.4 +charset-normalizer==2.0.6 click==8.0.1 confluent-kafka==1.7.0 flake8==3.9.2 flake8-quotes==3.3.0 -Flask==2.0.1 +Flask==2.0.2 gevent==21.8.0 -greenlet==1.1.1 +greenlet==1.1.2 idna==3.2 iniconfig==1.1.1 itsdangerous==2.0.1 -Jinja2==3.0.1 +Jinja2==3.0.2 MarkupSafe==2.0.1 mccabe==0.6.1 packaging==21.0 @@ -40,9 +40,9 @@ pytest==6.2.5 requests==2.26.0 six==1.16.0 spavro==1.1.24 -SQLAlchemy==1.4.23 +SQLAlchemy==1.4.25 toml==0.10.2 -urllib3==1.26.6 +urllib3==1.26.7 Werkzeug==2.0.1 zope.event==4.5.0 zope.interface==5.4.0 diff --git a/aether-ui/aether/ui/assets/package.json b/aether-ui/aether/ui/assets/package.json index 697b914c5..6a1dc955d 100644 --- a/aether-ui/aether/ui/assets/package.json +++ b/aether-ui/aether/ui/assets/package.json @@ -63,7 +63,7 @@ "stylelint": "~13.13.0", "stylelint-config-standard": "~22.0.0", "webpack": "~4.44.0", - "webpack-bundle-tracker": "~1.3.0", + "webpack-bundle-tracker": "~1.4.0", "webpack-cli": "~3.3.0", "webpack-dev-middleware": "~3.7.0", "webpack-hot-middleware": "~2.25.0" diff --git a/aether-ui/conf/pip/requirements.txt b/aether-ui/conf/pip/requirements.txt index 86eceb3d5..9c0c24126 100644 --- a/aether-ui/conf/pip/requirements.txt +++ b/aether-ui/conf/pip/requirements.txt @@ -15,18 +15,17 @@ aether.sdk==1.4.0 asgiref==3.4.1 autopep8==1.5.7 -boto3==1.18.40 -botocore==1.21.40 -cachetools==4.2.2 +boto3==1.18.54 +botocore==1.21.54 +cachetools==4.2.4 certifi==2021.5.30 -cffi==1.14.6 -charset-normalizer==2.0.4 +charset-normalizer==2.0.6 configparser==5.0.2 -coverage==5.5 -Django==3.2.7 +coverage==6.0 +Django==3.2.8 django-cacheops==6.0 django-cleanup==5.2.0 -django-cors-headers==3.8.0 +django-cors-headers==3.10.0 django-debug-toolbar==3.2.2 django-minio-storage==0.3.10 django-model-utils==4.1.1 @@ -36,49 +35,48 @@ django-redis==5.0.0 django-silk==4.1.0 django-storages==1.11.1 django-uwsgi==0.2.2 -django-webpack-loader==1.3.0 +django-webpack-loader==1.4.1 djangorestframework==3.12.4 drf-dynamic-fields==0.3.1 flake8==3.9.2 flake8-quotes==3.3.0 funcy==1.16 google-api-core==2.0.1 -google-auth==2.0.2 +google-auth==2.2.1 google-cloud-core==2.0.0 -google-cloud-storage==1.42.1 -google-crc32c==1.1.2 -google-resumable-media==2.0.2 +google-cloud-storage==1.42.3 +google-crc32c==1.2.0 +google-resumable-media==2.0.3 googleapis-common-protos==1.53.0 gprof2dot==2021.2.21 -greenlet==1.1.1 +greenlet==1.1.2 idna==3.2 -Jinja2==3.0.1 +Jinja2==3.0.2 jmespath==0.10.0 Markdown==3.3.4 MarkupSafe==2.0.1 mccabe==0.6.1 minio==6.0.2 prometheus-client==0.11.0 -protobuf==3.17.3 +protobuf==3.18.0 psycopg2-binary==2.9.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycodestyle==2.7.0 -pycparser==2.20 pyflakes==2.3.1 Pygments==2.10.0 python-dateutil==2.8.2 python-json-logger==2.0.2 -pytz==2021.1 +pytz==2021.3 redis==3.5.3 requests==2.26.0 rsa==4.7.2 s3transfer==0.5.0 -sentry-sdk==1.3.1 +sentry-sdk==1.4.3 six==1.16.0 -SQLAlchemy==1.4.23 +SQLAlchemy==1.4.25 sqlparse==0.4.2 tblib==1.7.0 toml==0.10.2 -urllib3==1.26.6 +urllib3==1.26.7 uWSGI==2.0.19.1 From 010c0d1d5ab41c76230915e4ca08a2f1bfb435c5 Mon Sep 17 00:00:00 2001 From: Obdulia Losantos Date: Tue, 5 Oct 2021 16:50:45 +0200 Subject: [PATCH 3/4] fix: wrong filter in admin extract endpoint (#964) * fix: wrong filter in admin extract endpoint * docs --- .../aether/kernel/api/entity_extractor.py | 32 +++++++++++-------- .../kernel/api/tests/test_entity_extractor.py | 26 +++++++++++++-- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/aether-kernel/aether/kernel/api/entity_extractor.py b/aether-kernel/aether/kernel/api/entity_extractor.py index 8a078fb49..5fb61a151 100644 --- a/aether-kernel/aether/kernel/api/entity_extractor.py +++ b/aether-kernel/aether/kernel/api/entity_extractor.py @@ -107,34 +107,38 @@ def extract(self, request, pk, *args, **kwargs): ) -@api_view(['POST']) +@api_view(['GET', 'POST']) @permission_classes([IsAdminUser]) @renderer_classes([JSONRenderer]) def extract_view(request, *args, **kwargs): ''' - Send to redis the submissions that are not yet extracted and - whose last modification time was not before the indicated delta (1 day). + Submit to redis the submissions that are not yet extracted and + whose last modification time was before the indicated delta (1 day). - Reachable at ``POST /admin/~extract?delta=1d`` + Reachable at ``GET|POST /admin/~extract?delta=1d&[submit]`` ''' delta = request.query_params.get('delta', '1d') modified = parse_delta(delta) - submissions = ( - Submission.objects - .filter(modified__lte=modified) - .filter(is_extracted=False) + submissions = Submission \ + .objects \ + .filter(modified__gte=modified) \ + .filter(project__active=True) \ .order_by('-modified') - .iterator()) - count = 0 - for submission in submissions: - count = count + 1 - send_model_item_to_redis(submission) + pending = submissions.filter(is_extracted=False) + + submitted = False + if request.method == 'POST' or 'submit' in request.query_params: + submitted = True + for submission in pending.iterator(): + send_model_item_to_redis(submission) return Response(data={ - 'count': count, + 'count': pending.count(), + 'total': submissions.count(), 'delta': delta, 'modified': modified, + 'submitted': submitted, 'timestamp': now(), }) diff --git a/aether-kernel/aether/kernel/api/tests/test_entity_extractor.py b/aether-kernel/aether/kernel/api/tests/test_entity_extractor.py index fba3f5171..d2027bf39 100644 --- a/aether-kernel/aether/kernel/api/tests/test_entity_extractor.py +++ b/aether-kernel/aether/kernel/api/tests/test_entity_extractor.py @@ -262,11 +262,31 @@ def test_admin_extract__endpoint(self): self.assertEqual(self.mappingset.submissions.count(), 101) self.assertEqual(self.mappingset.submissions.filter(is_extracted=False).count(), 101) - # request extraction + # check extraction with GET + response = self.client.get(url + '?delta=1w') + data = response.json() + self.assertEqual(data['delta'], '1w') + self.assertFalse(data['submitted']) + self.assertEqual(data['count'], 101) + + # request extraction with POST + with mock.patch('aether.kernel.api.entity_extractor.send_model_item_to_redis') as mock_fn_1: + response = self.client.post(url + '?delta=1w') + + mock_fn_1.assert_called() + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertEqual(data['delta'], '1w') + self.assertTrue(data['submitted']) + self.assertEqual(data['count'], 101) + + # force extraction with GET with mock.patch('aether.kernel.api.entity_extractor.send_model_item_to_redis') as mock_fn_1: - response = self.client.post(url + '?delta=0microseconds') + response = self.client.get(url + '?delta=1w&submit') mock_fn_1.assert_called() self.assertEqual(response.status_code, 200) data = response.json() - self.assertEqual(data['delta'], '0microseconds') + self.assertEqual(data['delta'], '1w') + self.assertTrue(data['submitted']) + self.assertEqual(data['total'], 101) From 1149444e571894fb9929a0c01d7343b79726e2ba Mon Sep 17 00:00:00 2001 From: Obdulia Losantos Date: Tue, 5 Oct 2021 17:19:23 +0200 Subject: [PATCH 4/4] feat: pending submissions in stats view (#965) --- .../aether/kernel/api/serializers.py | 8 +++++-- .../aether/kernel/api/tests/test_views.py | 24 ++++++++++++++++--- aether-kernel/aether/kernel/api/views.py | 8 +++++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/aether-kernel/aether/kernel/api/serializers.py b/aether-kernel/aether/kernel/api/serializers.py index 0a4908461..a6a698437 100644 --- a/aether-kernel/aether/kernel/api/serializers.py +++ b/aether-kernel/aether/kernel/api/serializers.py @@ -515,6 +515,7 @@ class ProjectStatsSerializer(DynamicFieldsMixin, DynamicFieldsModelSerializer): first_submission = serializers.DateTimeField(read_only=True) last_submission = serializers.DateTimeField(read_only=True) submissions_count = serializers.IntegerField(read_only=True) + pending_submissions_count = serializers.IntegerField(read_only=True) attachments_count = serializers.IntegerField(read_only=True) entities_count = serializers.IntegerField(read_only=True) @@ -523,7 +524,8 @@ class Meta: fields = ( 'id', 'name', 'created', 'active', 'first_submission', 'last_submission', - 'submissions_count', 'attachments_count', 'entities_count', + 'submissions_count', 'pending_submissions_count', + 'attachments_count', 'entities_count', ) @@ -532,6 +534,7 @@ class MappingSetStatsSerializer(DynamicFieldsMixin, DynamicFieldsModelSerializer first_submission = serializers.DateTimeField(read_only=True) last_submission = serializers.DateTimeField(read_only=True) submissions_count = serializers.IntegerField(read_only=True) + pending_submissions_count = serializers.IntegerField(read_only=True) attachments_count = serializers.IntegerField(read_only=True) entities_count = serializers.IntegerField(read_only=True) @@ -540,7 +543,8 @@ class Meta: fields = ( 'id', 'name', 'created', 'first_submission', 'last_submission', - 'submissions_count', 'attachments_count', 'entities_count', + 'submissions_count', 'pending_submissions_count', + 'attachments_count', 'entities_count', ) diff --git a/aether-kernel/aether/kernel/api/tests/test_views.py b/aether-kernel/aether/kernel/api/tests/test_views.py index 709066d73..fd5aa726d 100644 --- a/aether-kernel/aether/kernel/api/tests/test_views.py +++ b/aether-kernel/aether/kernel/api/tests/test_views.py @@ -185,27 +185,31 @@ def test_project_stats_view(self): data = response.json() self.assertEqual(data['submissions_count'], submissions_count) - # let's try again but with an unexistent family + # let's try again but with an inexistent family response = self.client.get(f'{url}?family=unknown') data = response.json() self.assertEqual(data['submissions_count'], submissions_count) + self.assertEqual(data['pending_submissions_count'], submissions_count) # let's try with using the project id response = self.client.get(f'{url}?family={str(self.project.pk)}') data = response.json() self.assertEqual(data['submissions_count'], submissions_count) + self.assertEqual(data['pending_submissions_count'], submissions_count) # let's try with the passthrough filter response = self.client.get(f'{url}?passthrough=true') data = response.json() self.assertEqual(data['submissions_count'], submissions_count) + self.assertEqual(data['pending_submissions_count'], submissions_count) - # delete the submissions and check the entities + # delete the submissions and check the count models.Submission.objects.all().delete() self.assertEqual(models.Submission.objects.count(), 0) response = self.client.get(url) data = response.json() self.assertEqual(data['submissions_count'], 0) + self.assertEqual(data['pending_submissions_count'], 0) def test_project_stats_view_fields(self): url = reverse('projects_stats-detail', kwargs={'pk': self.project.pk}) @@ -219,7 +223,16 @@ def test_project_stats_view_fields(self): response = self.client.get( url, - {'omit': 'created,first_submission,last_submission,submissions_count,entities_count'} + { + 'omit': ','.join([ + 'created', + 'first_submission', + 'last_submission', + 'submissions_count', + 'pending_submissions_count', + 'entities_count', + ]) + } ) data = response.json() self.assertIn('id', data) @@ -229,6 +242,7 @@ def test_project_stats_view_fields(self): self.assertNotIn('first_submission', data) self.assertNotIn('last_submission', data) self.assertNotIn('submissions_count', data) + self.assertNotIn('pending_submissions_count', data) self.assertNotIn('entities_count', data) response = self.client.get(url, {'fields': 'entities_count'}) @@ -240,6 +254,7 @@ def test_project_stats_view_fields(self): self.assertNotIn('first_submission', data) self.assertNotIn('last_submission', data) self.assertNotIn('submissions_count', data) + self.assertNotIn('pending_submissions_count', data) self.assertIn('entities_count', data) response = self.client.get( @@ -256,6 +271,7 @@ def test_project_stats_view_fields(self): self.assertNotIn('first_submission', data) self.assertNotIn('last_submission', data) self.assertNotIn('submissions_count', data) + self.assertNotIn('pending_submissions_count', data) self.assertIn('entities_count', data) def test_mapping_set_stats_view(self): @@ -266,6 +282,7 @@ def test_mapping_set_stats_view(self): self.assertEqual(data['id'], str(self.mappingset.pk)) submissions_count = models.Submission.objects.filter(mappingset=self.mappingset.pk).count() self.assertEqual(data['submissions_count'], submissions_count) + self.assertEqual(data['pending_submissions_count'], 0, 'All extracted') entities_count = models.Entity.objects.filter(submission__mappingset=self.mappingset.pk).count() self.assertEqual(data['entities_count'], entities_count) self.assertLessEqual( @@ -279,6 +296,7 @@ def test_mapping_set_stats_view(self): response = self.client.get(url) data = response.json() self.assertEqual(data['submissions_count'], 0) + self.assertEqual(data['pending_submissions_count'], 0) self.assertEqual(data['entities_count'], 0) def test_validate_mappings__success(self): diff --git a/aether-kernel/aether/kernel/api/views.py b/aether-kernel/aether/kernel/api/views.py index e1e5f652e..a44c133be 100644 --- a/aether-kernel/aether/kernel/api/views.py +++ b/aether-kernel/aether/kernel/api/views.py @@ -795,6 +795,14 @@ def _get_group_by_cols(*args): # pragma: no cover qs = qs.annotate( submissions_count=db_models.Count(f'{self.submissions_field}__id', distinct=True) ) + if _is_included('pending_submissions_count'): + qs = qs.annotate( + pending_submissions_count=db_models.Count( + f'{self.submissions_field}__id', + filter=db_models.Q(**{f'{self.submissions_field}__is_extracted': False}), + distinct=True, + ) + ) if _is_included('attachments_count'): qs = qs.annotate( attachments_count=db_models.Count(f'{self.submissions_field}__attachments__id', distinct=True)