Skip to content

Commit

Permalink
inconsistency number and debug errors fixed
Browse files Browse the repository at this point in the history
  • Loading branch information
@helllllllder
helllllllder committed Apr 14, 2022
1 parent d218eea commit 7ab6652
Show file tree
Hide file tree
Showing 4 changed files with 112 additions and 212 deletions.
109 changes: 4 additions & 105 deletions bothub/api/v2/examples/filters.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
from django.core.exceptions import ValidationError as DjangoValidationError
from django.db.models import Count
from django.db.models import Q, F
from django.db.models import Count, Q
from django.utils.translation import ugettext_lazy as _
from django_filters import rest_framework as filters
from rest_framework.exceptions import NotFound
Expand All @@ -9,8 +8,10 @@
from bothub.common.models import Repository
from bothub.common.models import RepositoryExample

from bothub.utils import DefaultExamplesFilter

class ExamplesFilter(filters.FilterSet):

class ExamplesFilter(DefaultExamplesFilter):
class Meta:
model = RepositoryExample
fields = ["text", "language"]
Expand Down Expand Up @@ -123,29 +124,6 @@ def filter_repository_version(self, queryset, name, value):
repository_version_language__repository_version__pk=value
)

def filter_has_translation(self, queryset, name, value):
annotated_queryset = queryset.annotate(translation_count=Count("translations"))
if value:
return annotated_queryset.filter(translation_count__gt=0)
else:
return annotated_queryset.filter(translation_count=0)

def filter_has_not_translation_to(self, queryset, name, value):
annotated_queryset = queryset.annotate(
translation_count=Count(
"translations", filter=Q(translations__language=value)
)
)
return annotated_queryset.filter(translation_count=0)

def filter_has_translation_to(self, queryset, name, value):
annotated_queryset = queryset.annotate(
translation_count=Count(
"translations", filter=Q(translations__language=value)
)
)
return annotated_queryset.filter(~Q(translation_count=0))

def filter_is_available_language(self, queryset, name, value):
annotated_queryset = queryset.annotate(
translation_count=Count(
Expand All @@ -168,82 +146,3 @@ def filter_order_by_translation(self, queryset, name, value):
"-translation_count" if inverted else "translation_count"
)
return result_queryset

def filter_group(self, queryset, name, value):
if value == "other":
return queryset.filter(entities__entity__group__isnull=True)
return queryset.filter(entities__entity__group__value=value)

def filter_entity(self, queryset, name, value):
return queryset.filter(entities__entity__value=value).distinct()

def filter_entity_id(self, queryset, name, value):
return queryset.filter(entities__entity__pk=value).distinct()

def filter_intent(self, queryset, name, value):
return queryset.filter(intent__text=value)

def filter_intent_id(self, queryset, name, value):
return queryset.filter(intent__pk=value)

def filter_has_valid_entities(self, queryset, name, value):
result_queryset = queryset.annotate(
original_entities_count=Count(
"entities",
filter=Q(
translations__original_example__entities__repository_example=F("pk")
)
& Q(translations__language=value),
distinct=True,
)
).annotate(
entities_count=Count(
"translations__entities",
filter=Q(
Q(
translations__entities__repository_translated_example__language=value
)
| Q(
translations__entities__repository_translated_example__language=F(
"repository_version_language__repository_version__repository__language"
)
),
translations__entities__entity__in=F(
"translations__original_example__entities__entity"
),
),
distinct=True,
)
)
return result_queryset.filter(original_entities_count=F("entities_count"))

def filter_has_invalid_entities(self, queryset, name, value):
result_queryset = queryset.annotate(
original_entities_count=Count(
"entities",
filter=Q(
translations__original_example__entities__repository_example=F("pk")
)
& Q(translations__language=value),
distinct=True,
)
).annotate(
entities_count=Count(
"translations__entities",
filter=Q(
Q(
translations__entities__repository_translated_example__language=value
)
| Q(
translations__entities__repository_translated_example__language=F(
"repository_version_language__repository_version__repository__language"
)
),
translations__entities__entity__in=F(
"translations__original_example__entities__entity"
),
),
distinct=True,
)
)
return result_queryset.exclude(original_entities_count=F("entities_count"))
13 changes: 12 additions & 1 deletion bothub/api/v2/nlp/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,10 +331,21 @@ def retrieve(self, request, *args, **kwargs):
repository = repository_authorization.repository

repository_version = request.query_params.get("repository_version")
repository_version_language = request.query_params.get(
"repository_version_language"
)

try:
is_default = RepositoryVersion.objects.get(pk=repository_version).is_default
except (RepositoryVersion.DoesNotExist, ValueError):
is_default = True
try:
repository_version_language = RepositoryVersionLanguage.objects.get(
pk=repository_version_language
)
is_default = repository_version_language.repository_version.is_default
repository_version = repository_version_language.repository_version.pk
except (RepositoryVersionLanguage.DoesNotExist, ValueError):
is_default = True
queryset = RepositoryExample.objects.filter(
repository_version_language__repository_version__repository=repository
)
Expand Down
109 changes: 4 additions & 105 deletions bothub/api/v2/translator/filters.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
from django.db.models import Count, F
from django.db.models import Q
from django.db.models import Count, Q
from django.utils.translation import ugettext_lazy as _
from django_filters import rest_framework as filters
from rest_framework.exceptions import PermissionDenied, NotFound
Expand All @@ -12,8 +11,10 @@
RepositoryTranslatedExample,
)

from bothub.utils import DefaultExamplesFilter

class RepositoryTranslatorFilter(filters.FilterSet):

class RepositoryTranslatorFilter(DefaultExamplesFilter):
class Meta:
model = RepositoryTranslator
fields = ["repository"]
Expand Down Expand Up @@ -115,108 +116,6 @@ def filter_order_by_translation(self, queryset, name, value):
)
return result_queryset

def filter_has_translation(self, queryset, name, value):
annotated_queryset = queryset.annotate(translation_count=Count("translations"))
if value:
return annotated_queryset.filter(translation_count__gt=0)
else:
return annotated_queryset.filter(translation_count=0)

def filter_has_not_translation_to(self, queryset, name, value):
annotated_queryset = queryset.annotate(
translation_count=Count(
"translations", filter=Q(translations__language=value)
)
)
return annotated_queryset.filter(translation_count=0)

def filter_has_translation_to(self, queryset, name, value):
annotated_queryset = queryset.annotate(
translation_count=Count(
"translations", filter=Q(translations__language=value)
)
)
return annotated_queryset.filter(~Q(translation_count=0))

def filter_group(self, queryset, name, value):
if value == "other":
return queryset.filter(entities__entity__group__isnull=True)
return queryset.filter(entities__entity__group__value=value)

def filter_entity(self, queryset, name, value):
return queryset.filter(entities__entity__value=value).distinct()

def filter_entity_id(self, queryset, name, value):
return queryset.filter(entities__entity__pk=value).distinct()

def filter_intent(self, queryset, name, value):
return queryset.filter(intent__text=value)

def filter_intent_id(self, queryset, name, value):
return queryset.filter(intent__pk=value)

def filter_has_valid_entities(self, queryset, name, value):
result_queryset = queryset.annotate(
original_entities_count=Count(
"entities",
filter=Q(
translations__original_example__entities__repository_example=F("pk")
)
& Q(translations__language=value),
distinct=True,
)
).annotate(
entities_count=Count(
"translations__entities",
filter=Q(
Q(
translations__entities__repository_translated_example__language=value
)
| Q(
translations__entities__repository_translated_example__language=F(
"repository_version_language__repository_version__repository__language"
)
),
translations__entities__entity__in=F(
"translations__original_example__entities__entity"
),
),
distinct=True,
)
)
return result_queryset.filter(original_entities_count=F("entities_count"))

def filter_has_invalid_entities(self, queryset, name, value):
result_queryset = queryset.annotate(
original_entities_count=Count(
"entities",
filter=Q(
translations__original_example__entities__repository_example=F("pk")
)
& Q(translations__language=value),
distinct=True,
)
).annotate(
entities_count=Count(
"translations__entities",
filter=Q(
Q(
translations__entities__repository_translated_example__language=value
)
| Q(
translations__entities__repository_translated_example__language=F(
"repository_version_language__repository_version__repository__language"
)
),
translations__entities__entity__in=F(
"translations__original_example__entities__entity"
),
),
distinct=True,
)
)
return result_queryset.exclude(original_entities_count=F("entities_count"))


class TranslationsTranslatorFilter(filters.FilterSet):
class Meta:
Expand Down
93 changes: 92 additions & 1 deletion bothub/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@
from collections import OrderedDict
from botocore.exceptions import ClientError
from django.conf import settings
from django.db.models import IntegerField, Subquery
from django.db.models import IntegerField, Subquery, Q, F, Count
from django.utils.text import slugify
from django.utils.translation import ugettext_lazy as _
from django_elasticsearch_dsl import Document
from django_filters import rest_framework as filters
from rest_framework import status
from rest_framework.exceptions import APIException, ValidationError

Expand Down Expand Up @@ -371,3 +373,92 @@ def get_organization(request, organization_id: int):
request.context.abort(
grpc.StatusCode.NOT_FOUND, f"{organization_id} not found!"
)


class TimeBasedDocument(Document):
def save(self, action="create", **kwargs):
return super().save(action=action, **kwargs)

def update(self, instance, action="create", **kwargs):
return super().update(instance, action=action, **kwargs)


def filter_validate_entities(queryset, value):
entities = list(
queryset.values_list("entities__entity", flat=True).order_by().distinct()
)

result_queryset = queryset.annotate(
original_entities_count=Count(
"entities",
filter=Q(translations__language=value),
distinct=True,
)
).annotate(
entities_count=Count(
"translations__entities",
filter=Q(
Q(translations__entities__repository_translated_example__language=value)
| Q(
translations__entities__repository_translated_example__language=F(
"repository_version_language__repository_version__repository__language"
)
),
translations__entities__entity__in=entities,
),
distinct=True,
)
)
return result_queryset


class DefaultExamplesFilter(filters.FilterSet):
def filter_has_translation(self, queryset, name, value):
annotated_queryset = queryset.annotate(translation_count=Count("translations"))
if value:
return annotated_queryset.filter(translation_count__gt=0)
else:
return annotated_queryset.filter(translation_count=0)

def filter_has_not_translation_to(self, queryset, name, value):
annotated_queryset = queryset.annotate(
translation_count=Count(
"translations", filter=Q(translations__language=value)
)
)
return annotated_queryset.filter(translation_count=0)

def filter_has_translation_to(self, queryset, name, value):
annotated_queryset = queryset.annotate(
translation_count=Count(
"translations", filter=Q(translations__language=value)
)
)
return annotated_queryset.filter(~Q(translation_count=0))

def filter_group(self, queryset, name, value):
if value == "other":
return queryset.filter(entities__entity__group__isnull=True)
return queryset.filter(entities__entity__group__value=value)

def filter_entity(self, queryset, name, value):
return queryset.filter(entities__entity__value=value).distinct()

def filter_entity_id(self, queryset, name, value):
return queryset.filter(entities__entity__pk=value).distinct()

def filter_intent(self, queryset, name, value):
return queryset.filter(intent__text=value)

def filter_intent_id(self, queryset, name, value):
return queryset.filter(intent__pk=value)

def filter_has_valid_entities(self, queryset, name, value):
return filter_validate_entities(queryset, value).filter(
original_entities_count=F("entities_count")
)

def filter_has_invalid_entities(self, queryset, name, value):
return filter_validate_entities(queryset, value).exclude(
original_entities_count=F("entities_count")
)

0 comments on commit 7ab6652

Please sign in to comment.