diff --git a/bothub/api/v2/examples/filters.py b/bothub/api/v2/examples/filters.py index 985e37a1..3299c6c2 100644 --- a/bothub/api/v2/examples/filters.py +++ b/bothub/api/v2/examples/filters.py @@ -1,6 +1,5 @@ from django.core.exceptions import ValidationError as DjangoValidationError -from django.db.models import Count -from django.db.models import Q, F +from django.db.models import Count, Q from django.utils.translation import ugettext_lazy as _ from django_filters import rest_framework as filters from rest_framework.exceptions import NotFound @@ -9,8 +8,10 @@ from bothub.common.models import Repository from bothub.common.models import RepositoryExample +from bothub.utils import DefaultExamplesFilter -class ExamplesFilter(filters.FilterSet): + +class ExamplesFilter(DefaultExamplesFilter): class Meta: model = RepositoryExample fields = ["text", "language"] @@ -123,29 +124,6 @@ def filter_repository_version(self, queryset, name, value): repository_version_language__repository_version__pk=value ) - def filter_has_translation(self, queryset, name, value): - annotated_queryset = queryset.annotate(translation_count=Count("translations")) - if value: - return annotated_queryset.filter(translation_count__gt=0) - else: - return annotated_queryset.filter(translation_count=0) - - def filter_has_not_translation_to(self, queryset, name, value): - annotated_queryset = queryset.annotate( - translation_count=Count( - "translations", filter=Q(translations__language=value) - ) - ) - return annotated_queryset.filter(translation_count=0) - - def filter_has_translation_to(self, queryset, name, value): - annotated_queryset = queryset.annotate( - translation_count=Count( - "translations", filter=Q(translations__language=value) - ) - ) - return annotated_queryset.filter(~Q(translation_count=0)) - def filter_is_available_language(self, queryset, name, value): annotated_queryset = queryset.annotate( translation_count=Count( @@ -168,82 +146,3 @@ def filter_order_by_translation(self, queryset, name, value): "-translation_count" if inverted else "translation_count" ) return result_queryset - - def filter_group(self, queryset, name, value): - if value == "other": - return queryset.filter(entities__entity__group__isnull=True) - return queryset.filter(entities__entity__group__value=value) - - def filter_entity(self, queryset, name, value): - return queryset.filter(entities__entity__value=value).distinct() - - def filter_entity_id(self, queryset, name, value): - return queryset.filter(entities__entity__pk=value).distinct() - - def filter_intent(self, queryset, name, value): - return queryset.filter(intent__text=value) - - def filter_intent_id(self, queryset, name, value): - return queryset.filter(intent__pk=value) - - def filter_has_valid_entities(self, queryset, name, value): - result_queryset = queryset.annotate( - original_entities_count=Count( - "entities", - filter=Q( - translations__original_example__entities__repository_example=F("pk") - ) - & Q(translations__language=value), - distinct=True, - ) - ).annotate( - entities_count=Count( - "translations__entities", - filter=Q( - Q( - translations__entities__repository_translated_example__language=value - ) - | Q( - translations__entities__repository_translated_example__language=F( - "repository_version_language__repository_version__repository__language" - ) - ), - translations__entities__entity__in=F( - "translations__original_example__entities__entity" - ), - ), - distinct=True, - ) - ) - return result_queryset.filter(original_entities_count=F("entities_count")) - - def filter_has_invalid_entities(self, queryset, name, value): - result_queryset = queryset.annotate( - original_entities_count=Count( - "entities", - filter=Q( - translations__original_example__entities__repository_example=F("pk") - ) - & Q(translations__language=value), - distinct=True, - ) - ).annotate( - entities_count=Count( - "translations__entities", - filter=Q( - Q( - translations__entities__repository_translated_example__language=value - ) - | Q( - translations__entities__repository_translated_example__language=F( - "repository_version_language__repository_version__repository__language" - ) - ), - translations__entities__entity__in=F( - "translations__original_example__entities__entity" - ), - ), - distinct=True, - ) - ) - return result_queryset.exclude(original_entities_count=F("entities_count")) diff --git a/bothub/api/v2/nlp/views.py b/bothub/api/v2/nlp/views.py index 67be00e7..a6758bb3 100644 --- a/bothub/api/v2/nlp/views.py +++ b/bothub/api/v2/nlp/views.py @@ -331,10 +331,21 @@ def retrieve(self, request, *args, **kwargs): repository = repository_authorization.repository repository_version = request.query_params.get("repository_version") + repository_version_language = request.query_params.get( + "repository_version_language" + ) + try: is_default = RepositoryVersion.objects.get(pk=repository_version).is_default except (RepositoryVersion.DoesNotExist, ValueError): - is_default = True + try: + repository_version_language = RepositoryVersionLanguage.objects.get( + pk=repository_version_language + ) + is_default = repository_version_language.repository_version.is_default + repository_version = repository_version_language.repository_version.pk + except (RepositoryVersionLanguage.DoesNotExist, ValueError): + is_default = True queryset = RepositoryExample.objects.filter( repository_version_language__repository_version__repository=repository ) diff --git a/bothub/api/v2/translator/filters.py b/bothub/api/v2/translator/filters.py index 20082647..f17d94a6 100644 --- a/bothub/api/v2/translator/filters.py +++ b/bothub/api/v2/translator/filters.py @@ -1,5 +1,4 @@ -from django.db.models import Count, F -from django.db.models import Q +from django.db.models import Count, Q from django.utils.translation import ugettext_lazy as _ from django_filters import rest_framework as filters from rest_framework.exceptions import PermissionDenied, NotFound @@ -12,8 +11,10 @@ RepositoryTranslatedExample, ) +from bothub.utils import DefaultExamplesFilter -class RepositoryTranslatorFilter(filters.FilterSet): + +class RepositoryTranslatorFilter(DefaultExamplesFilter): class Meta: model = RepositoryTranslator fields = ["repository"] @@ -115,108 +116,6 @@ def filter_order_by_translation(self, queryset, name, value): ) return result_queryset - def filter_has_translation(self, queryset, name, value): - annotated_queryset = queryset.annotate(translation_count=Count("translations")) - if value: - return annotated_queryset.filter(translation_count__gt=0) - else: - return annotated_queryset.filter(translation_count=0) - - def filter_has_not_translation_to(self, queryset, name, value): - annotated_queryset = queryset.annotate( - translation_count=Count( - "translations", filter=Q(translations__language=value) - ) - ) - return annotated_queryset.filter(translation_count=0) - - def filter_has_translation_to(self, queryset, name, value): - annotated_queryset = queryset.annotate( - translation_count=Count( - "translations", filter=Q(translations__language=value) - ) - ) - return annotated_queryset.filter(~Q(translation_count=0)) - - def filter_group(self, queryset, name, value): - if value == "other": - return queryset.filter(entities__entity__group__isnull=True) - return queryset.filter(entities__entity__group__value=value) - - def filter_entity(self, queryset, name, value): - return queryset.filter(entities__entity__value=value).distinct() - - def filter_entity_id(self, queryset, name, value): - return queryset.filter(entities__entity__pk=value).distinct() - - def filter_intent(self, queryset, name, value): - return queryset.filter(intent__text=value) - - def filter_intent_id(self, queryset, name, value): - return queryset.filter(intent__pk=value) - - def filter_has_valid_entities(self, queryset, name, value): - result_queryset = queryset.annotate( - original_entities_count=Count( - "entities", - filter=Q( - translations__original_example__entities__repository_example=F("pk") - ) - & Q(translations__language=value), - distinct=True, - ) - ).annotate( - entities_count=Count( - "translations__entities", - filter=Q( - Q( - translations__entities__repository_translated_example__language=value - ) - | Q( - translations__entities__repository_translated_example__language=F( - "repository_version_language__repository_version__repository__language" - ) - ), - translations__entities__entity__in=F( - "translations__original_example__entities__entity" - ), - ), - distinct=True, - ) - ) - return result_queryset.filter(original_entities_count=F("entities_count")) - - def filter_has_invalid_entities(self, queryset, name, value): - result_queryset = queryset.annotate( - original_entities_count=Count( - "entities", - filter=Q( - translations__original_example__entities__repository_example=F("pk") - ) - & Q(translations__language=value), - distinct=True, - ) - ).annotate( - entities_count=Count( - "translations__entities", - filter=Q( - Q( - translations__entities__repository_translated_example__language=value - ) - | Q( - translations__entities__repository_translated_example__language=F( - "repository_version_language__repository_version__repository__language" - ) - ), - translations__entities__entity__in=F( - "translations__original_example__entities__entity" - ), - ), - distinct=True, - ) - ) - return result_queryset.exclude(original_entities_count=F("entities_count")) - class TranslationsTranslatorFilter(filters.FilterSet): class Meta: diff --git a/bothub/utils.py b/bothub/utils.py index 5da8e638..38949d12 100644 --- a/bothub/utils.py +++ b/bothub/utils.py @@ -12,9 +12,11 @@ from collections import OrderedDict from botocore.exceptions import ClientError from django.conf import settings -from django.db.models import IntegerField, Subquery +from django.db.models import IntegerField, Subquery, Q, F, Count from django.utils.text import slugify from django.utils.translation import ugettext_lazy as _ +from django_elasticsearch_dsl import Document +from django_filters import rest_framework as filters from rest_framework import status from rest_framework.exceptions import APIException, ValidationError @@ -371,3 +373,92 @@ def get_organization(request, organization_id: int): request.context.abort( grpc.StatusCode.NOT_FOUND, f"{organization_id} not found!" ) + + +class TimeBasedDocument(Document): + def save(self, action="create", **kwargs): + return super().save(action=action, **kwargs) + + def update(self, instance, action="create", **kwargs): + return super().update(instance, action=action, **kwargs) + + +def filter_validate_entities(queryset, value): + entities = list( + queryset.values_list("entities__entity", flat=True).order_by().distinct() + ) + + result_queryset = queryset.annotate( + original_entities_count=Count( + "entities", + filter=Q(translations__language=value), + distinct=True, + ) + ).annotate( + entities_count=Count( + "translations__entities", + filter=Q( + Q(translations__entities__repository_translated_example__language=value) + | Q( + translations__entities__repository_translated_example__language=F( + "repository_version_language__repository_version__repository__language" + ) + ), + translations__entities__entity__in=entities, + ), + distinct=True, + ) + ) + return result_queryset + + +class DefaultExamplesFilter(filters.FilterSet): + def filter_has_translation(self, queryset, name, value): + annotated_queryset = queryset.annotate(translation_count=Count("translations")) + if value: + return annotated_queryset.filter(translation_count__gt=0) + else: + return annotated_queryset.filter(translation_count=0) + + def filter_has_not_translation_to(self, queryset, name, value): + annotated_queryset = queryset.annotate( + translation_count=Count( + "translations", filter=Q(translations__language=value) + ) + ) + return annotated_queryset.filter(translation_count=0) + + def filter_has_translation_to(self, queryset, name, value): + annotated_queryset = queryset.annotate( + translation_count=Count( + "translations", filter=Q(translations__language=value) + ) + ) + return annotated_queryset.filter(~Q(translation_count=0)) + + def filter_group(self, queryset, name, value): + if value == "other": + return queryset.filter(entities__entity__group__isnull=True) + return queryset.filter(entities__entity__group__value=value) + + def filter_entity(self, queryset, name, value): + return queryset.filter(entities__entity__value=value).distinct() + + def filter_entity_id(self, queryset, name, value): + return queryset.filter(entities__entity__pk=value).distinct() + + def filter_intent(self, queryset, name, value): + return queryset.filter(intent__text=value) + + def filter_intent_id(self, queryset, name, value): + return queryset.filter(intent__pk=value) + + def filter_has_valid_entities(self, queryset, name, value): + return filter_validate_entities(queryset, value).filter( + original_entities_count=F("entities_count") + ) + + def filter_has_invalid_entities(self, queryset, name, value): + return filter_validate_entities(queryset, value).exclude( + original_entities_count=F("entities_count") + )