diff --git a/Makefile b/Makefile index 769f305ff..115c896b6 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ install_requirements: lint: @make development_mode_guard @make check_environment - @pipenv run flake8 + @PIPENV_DONT_LOAD_ENV=1 pipenv run flake8 @echo "${SUCCESS}✔${NC} The code is following the PEP8" test: @@ -92,6 +92,7 @@ development_mode_guard: _check_environment: @type pipenv &> /dev/null || (echo "${DANGER}☓${NC} Install pipenv to continue..." && exit 1) @echo "${SUCCESS}✔${NC} pipenv installed" - @if [[ ! -f "${ENVIRONMENT_VARS_FILE}" && ${IS_PRODUCTION} = false ]]; then make create_environment_vars_file; fi + @if [[ ! -f "${ENVIRONMENT_VARS_FILE}" && ${IS_PRODUCTION} = false ]]; \ + then make create_environment_vars_file; fi @make install_requirements @echo "${SUCCESS}✔${NC} Environment checked" \ No newline at end of file diff --git a/bothub/api/fields.py b/bothub/api/fields.py index e0ce136a3..b41552d3d 100644 --- a/bothub/api/fields.py +++ b/bothub/api/fields.py @@ -1,5 +1,8 @@ from rest_framework import serializers +from bothub.common.models import RepositoryEntity +from bothub.common.models import RepositoryEntityLabel + class ModelMultipleChoiceField(serializers.ManyRelatedField): pass @@ -17,3 +20,37 @@ def __init__(self, *args, **kwargs): class EntityText(serializers.CharField): pass + + +class EntityValueField(serializers.CharField): + def __init__(self, *args, validators=[], **kwargs): + kwargs.pop('max_length', 0) + kwargs.pop('help_text', '') + + value_field = RepositoryEntity._meta.get_field('value') + + super().__init__( + *args, + max_length=value_field.max_length, + validators=(validators + value_field.validators), + **kwargs) + + def to_representation(self, obj): + return obj.value + + +class LabelValueField(serializers.CharField): + def __init__(self, *args, validators=[], **kwargs): + kwargs.pop('max_length', 0) + kwargs.pop('help_text', '') + + value_field = RepositoryEntityLabel._meta.get_field('value') + + super().__init__( + *args, + max_length=value_field.max_length, + validators=(validators + value_field.validators), + **kwargs) + + def to_representation(self, obj): + return obj.value diff --git a/bothub/api/routers.py b/bothub/api/routers.py index 853655a8b..f038c2afc 100644 --- a/bothub/api/routers.py +++ b/bothub/api/routers.py @@ -7,8 +7,6 @@ from .views import RepositoryExampleViewSet from .views import NewRepositoryTranslatedExampleViewSet from .views import RepositoryTranslatedExampleViewSet -from .views import NewRepositoryTranslatedExampleEntityViewSet -from .views import RepositoryTranslatedExampleEntityViewSet from .views import RepositoryExamplesViewSet from .views import RegisterUserViewSet from .views import LoginViewSet @@ -26,6 +24,7 @@ from .views import RequestAuthorizationViewSet from .views import RepositoryAuthorizationRequestsViewSet from .views import ReviewAuthorizationRequestViewSet +from .views import RepositoryEntitiesViewSet class Router(routers.SimpleRouter): @@ -101,10 +100,6 @@ def get_lookup_regex(self, viewset, lookup_prefix=''): router.register('example', RepositoryExampleViewSet) router.register('translate-example', NewRepositoryTranslatedExampleViewSet) router.register('translation', RepositoryTranslatedExampleViewSet) -router.register('translation-entity/new', - NewRepositoryTranslatedExampleEntityViewSet) -router.register('translation-entity', - RepositoryTranslatedExampleEntityViewSet) router.register('examples', RepositoryExamplesViewSet) router.register('register', RegisterUserViewSet) router.register('login', LoginViewSet) @@ -125,3 +120,4 @@ def get_lookup_regex(self, viewset, lookup_prefix=''): RepositoryAuthorizationRequestsViewSet) router.register('review-authorization-request', ReviewAuthorizationRequestViewSet) +router.register('entities', RepositoryEntitiesViewSet) diff --git a/bothub/api/serializers/__init__.py b/bothub/api/serializers/__init__.py index 0623947d6..a28c6147f 100644 --- a/bothub/api/serializers/__init__.py +++ b/bothub/api/serializers/__init__.py @@ -17,6 +17,7 @@ RepositoryExampleSerializer, NewRepositoryExampleSerializer, NewRepositoryExampleEntitySerializer, + RepositoryEntitySerializer, ) from .translate import ( # noqa: F401 RepositoryTranslatedExampleEntitySeralizer, diff --git a/bothub/api/serializers/example.py b/bothub/api/serializers/example.py index c6086fdc2..cff4bd28e 100644 --- a/bothub/api/serializers/example.py +++ b/bothub/api/serializers/example.py @@ -1,15 +1,20 @@ from rest_framework import serializers +from rest_framework.fields import empty from django.utils.translation import gettext as _ from bothub.common.models import Repository from bothub.common.models import RepositoryExample from bothub.common.models import RepositoryExampleEntity +from bothub.common.models import RepositoryEntity from ..fields import EntityText +from ..fields import EntityValueField +from ..fields import LabelValueField from ..validators import CanContributeInRepositoryExampleValidator from ..validators import CanContributeInRepositoryValidator from ..validators import ExampleWithIntentOrEntityValidator +from ..validators import EntityNotEqualLabelValidator from .translate import RepositoryTranslatedExampleSerializer @@ -22,6 +27,7 @@ class Meta: 'start', 'end', 'entity', + 'label', 'created_at', 'value', ] @@ -32,10 +38,16 @@ class Meta: CanContributeInRepositoryExampleValidator(), ], help_text=_('Example\'s ID')) - value = serializers.SerializerMethodField() + entity = serializers.SerializerMethodField() + label = serializers.SerializerMethodField() - def get_value(self, obj): - return obj.value + def get_entity(self, obj): + return obj.entity.value + + def get_label(self, obj): + if not obj.entity.label: + return None + return obj.entity.label.value class NewRepositoryExampleEntitySerializer(serializers.ModelSerializer): @@ -46,8 +58,34 @@ class Meta: 'start', 'end', 'entity', + 'label', ] + repository_example = serializers.PrimaryKeyRelatedField( + queryset=RepositoryExample.objects, + required=False) + + entity = EntityValueField() + label = LabelValueField( + allow_blank=True, + required=False) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.validators.append(EntityNotEqualLabelValidator()) + + def create(self, validated_data): + repository_example = validated_data.pop('repository_example', None) + assert repository_example + label = validated_data.pop('label', empty) + example_entity = self.Meta.model.objects.create( + repository_example=repository_example, + **validated_data) + if label is not empty: + example_entity.entity.set_label(label) + example_entity.entity.save(update_fields=['label']) + return example_entity + class RepositoryExampleSerializer(serializers.ModelSerializer): class Meta: @@ -121,7 +159,26 @@ def create(self, validated_data): entities_data = validated_data.pop('entities') example = self.Meta.model.objects.create(**validated_data) for entity_data in entities_data: - RepositoryExampleEntity.objects.create( - repository_example=example, - **entity_data) + entity_data.update({'repository_example': example.pk}) + entity_serializer = NewRepositoryExampleEntitySerializer( + data=entity_data) + entity_serializer.is_valid(raise_exception=True) + entity_serializer.save() return example + + +class RepositoryEntitySerializer(serializers.ModelSerializer): + class Meta: + model = RepositoryEntity + fields = [ + 'repository', + 'value', + 'label', + ] + + label = serializers.SerializerMethodField() + + def get_label(self, obj): + if not obj.label: + return None + return obj.label.value diff --git a/bothub/api/serializers/repository.py b/bothub/api/serializers/repository.py index acfe33271..7120f1f19 100644 --- a/bothub/api/serializers/repository.py +++ b/bothub/api/serializers/repository.py @@ -87,6 +87,7 @@ class Meta: slug_field='nickname', read_only=True) categories_list = serializers.SerializerMethodField() + entities = serializers.SerializerMethodField() authorization = serializers.SerializerMethodField() examples__count = serializers.SerializerMethodField() request_authorization = serializers.SerializerMethodField() @@ -95,6 +96,9 @@ class Meta: def get_categories_list(self, obj): return RepositoryCategorySerializer(obj.categories, many=True).data + def get_entities(self, obj): + return obj.entities_list + def get_authorization(self, obj): request = self.context.get('request') if not request: diff --git a/bothub/api/serializers/translate.py b/bothub/api/serializers/translate.py index 4eab2fc3d..01fa9ce01 100644 --- a/bothub/api/serializers/translate.py +++ b/bothub/api/serializers/translate.py @@ -7,6 +7,7 @@ from bothub.common.models import RepositoryExample from bothub.common.languages import LANGUAGE_CHOICES +from ..fields import EntityValueField from ..validators import CanContributeInRepositoryTranslatedExampleValidator from ..validators import CanContributeInRepositoryExampleValidator from ..validators import TranslatedExampleEntitiesValidator @@ -32,8 +33,12 @@ class Meta: CanContributeInRepositoryTranslatedExampleValidator(), ], help_text='Example translation ID') + entity = serializers.SerializerMethodField() value = serializers.SerializerMethodField() + def get_entity(self, obj): + return obj.entity.value + def get_value(self, obj): return obj.value @@ -81,6 +86,8 @@ class Meta: 'entity', ] + entity = EntityValueField() + class NewRepositoryTranslatedExampleSerializer(serializers.ModelSerializer): class Meta: diff --git a/bothub/api/tests/test_example.py b/bothub/api/tests/test_example.py index 91f1ecf07..b034d4842 100644 --- a/bothub/api/tests/test_example.py +++ b/bothub/api/tests/test_example.py @@ -12,6 +12,7 @@ from ..views import NewRepositoryExampleViewSet from ..views import RepositoryExampleViewSet +from ..views import RepositoryEntitiesViewSet from .utils import create_user_and_token @@ -141,6 +142,87 @@ def test_with_entities(self): len(content_data.get('entities')), 1) + def test_with_entities_with_label(self): + response, content_data = self.request( + self.owner_token, + { + 'repository': str(self.repository.uuid), + 'text': 'my name is douglas', + 'intent': 'greet', + 'entities': [ + { + 'start': 11, + 'end': 18, + 'entity': 'name', + 'label': 'subject', + }, + ], + }) + self.assertEqual( + response.status_code, + status.HTTP_201_CREATED) + self.assertEqual( + len(content_data.get('entities')), + 1) + id = content_data.get('id') + repository_example = RepositoryExample.objects.get(id=id) + example_entity = repository_example.entities.all()[0] + self.assertIsNotNone(example_entity.entity.label) + + def test_with_entities_with_invalid_label(self): + response, content_data = self.request( + self.owner_token, + { + 'repository': str(self.repository.uuid), + 'text': 'my name is douglas', + 'intent': 'greet', + 'entities': [ + { + 'start': 11, + 'end': 18, + 'entity': 'name', + 'label': 'other', + }, + ], + }) + self.assertEqual( + response.status_code, + status.HTTP_400_BAD_REQUEST) + self.assertIn( + 'entities', + content_data.keys()) + entities_errors = content_data.get('entities') + self.assertIn( + 'label', + entities_errors[0]) + + def test_with_entities_with_equal_label(self): + response, content_data = self.request( + self.owner_token, + { + 'repository': str(self.repository.uuid), + 'text': 'my name is douglas', + 'intent': 'greet', + 'entities': [ + { + 'start': 11, + 'end': 18, + 'entity': 'name', + 'label': 'name', + }, + ], + }) + self.assertEqual( + response.status_code, + status.HTTP_400_BAD_REQUEST) + self.assertIn( + 'entities', + content_data.keys()) + entities_errors = content_data.get('entities') + self.assertIn( + 'label', + entities_errors[0]) + def test_intent_or_entity_required(self): response, content_data = self.request( self.owner_token, @@ -208,7 +290,7 @@ def setUp(self): self.example = RepositoryExample.objects.create( repository_update=self.repository.current_update(), text='my name is douglas') - RepositoryExampleEntity.objects.create( + self.example_entity = RepositoryExampleEntity.objects.create( repository_example=self.example, start=11, end=18, @@ -278,6 +360,36 @@ def test_list_entities(self): len(content_data.get('entities')), 1) + def test_entity_has_label(self): + response, content_data = self.request( + self.example, + self.owner_token) + self.assertEqual( + response.status_code, + status.HTTP_200_OK) + entity = content_data.get('entities')[0] + self.assertIn( + 'label', + entity.keys()) + + def test_entity_has_valid_label(self): + label = 'subject' + self.example_entity.entity.set_label('subject') + self.example_entity.entity.save(update_fields=['label']) + response, content_data = self.request( + self.example, + self.owner_token) + self.assertEqual( + response.status_code, + status.HTTP_200_OK) + entity = content_data.get('entities')[0] + self.assertIn( + 'label', + entity.keys()) + self.assertEqual( + entity.get('label'), + label) + class RepositoryExampleDestroyTestCase(TestCase): def setUp(self): @@ -357,3 +469,76 @@ def test_already_deleted(self): self.assertEqual( response.status_code, status.HTTP_500_INTERNAL_SERVER_ERROR) + + +class RepositoryEntitiesTestCase(TestCase): + def setUp(self): + self.factory = RequestFactory() + + self.owner, self.owner_token = create_user_and_token('owner') + self.user, self.user_token = create_user_and_token() + + self.entity_value = 'douglas' + + self.repository = Repository.objects.create( + owner=self.owner, + name='Testing', + slug='test', + language=languages.LANGUAGE_EN) + self.example = RepositoryExample.objects.create( + repository_update=self.repository.current_update(), + text='my name is douglas') + self.example_entity = RepositoryExampleEntity.objects.create( + repository_example=self.example, + start=11, + end=18, + entity=self.entity_value) + self.example_entity.entity.set_label('name') + self.example_entity.entity.save() + + def request(self, data, token): + authorization_header = { + 'HTTP_AUTHORIZATION': 'Token {}'.format(token.key), + } + request = self.factory.get( + '/api/entities/', + data=data, + **authorization_header) + response = RepositoryEntitiesViewSet.as_view( + {'get': 'list'})(request) + response.render() + content_data = json.loads(response.content) + return (response, content_data,) + + def test_okay(self): + response, content_data = self.request( + { + 'repository_uuid': self.repository.uuid, + }, + self.owner_token) + self.assertEqual( + response.status_code, + status.HTTP_200_OK) + self.assertEqual(content_data.get('count'), 1) + + response, content_data = self.request( + { + 'repository_uuid': self.repository.uuid, + 'value': self.entity_value, + }, + self.owner_token) + self.assertEqual( + response.status_code, + status.HTTP_200_OK) + self.assertEqual(content_data.get('count'), 1) + + response, content_data = self.request( + { + 'repository_uuid': self.repository.uuid, + 'value': 'other', + }, + self.owner_token) + self.assertEqual( + response.status_code, + status.HTTP_200_OK) + self.assertEqual(content_data.get('count'), 0) diff --git a/bothub/api/tests/test_translate.py b/bothub/api/tests/test_translate.py index 8b8d64c13..937a24964 100644 --- a/bothub/api/tests/test_translate.py +++ b/bothub/api/tests/test_translate.py @@ -9,13 +9,10 @@ from bothub.common.models import Repository from bothub.common.models import RepositoryExample from bothub.common.models import RepositoryTranslatedExample -from bothub.common.models import RepositoryTranslatedExampleEntity from bothub.common.models import RepositoryExampleEntity from ..views import NewRepositoryTranslatedExampleViewSet from ..views import RepositoryTranslatedExampleViewSet -from ..views import NewRepositoryTranslatedExampleEntityViewSet -from ..views import RepositoryTranslatedExampleEntityViewSet from ..views import TranslationsViewSet from .utils import create_user_and_token @@ -343,217 +340,6 @@ def test_forbidden(self): status.HTTP_403_FORBIDDEN) -class NewRepositoryTranslatedExampleEntityTestCase(TestCase): - def setUp(self): - self.factory = RequestFactory() - - self.owner, self.owner_token = create_user_and_token('owner') - - self.repository = Repository.objects.create( - owner=self.owner, - name='Testing', - slug='test', - language=languages.LANGUAGE_EN) - self.example = RepositoryExample.objects.create( - repository_update=self.repository.current_update(), - text='my name is Douglas') - self.translated = RepositoryTranslatedExample.objects.create( - original_example=self.example, - language=languages.LANGUAGE_PT, - text='meu nome é Douglas') - - def request(self, data, token): - authorization_header = { - 'HTTP_AUTHORIZATION': 'Token {}'.format(token.key), - } - request = self.factory.post( - '/api/translate-example/', - data, - **authorization_header) - response = NewRepositoryTranslatedExampleEntityViewSet.as_view( - {'post': 'create'})(request) - response.render() - content_data = json.loads(response.content) - return (response, content_data,) - - def test_okay(self): - response, content_data = self.request( - { - 'repository_translated_example': self.translated.id, - 'start': 11, - 'end': 18, - 'entity': 'name', - }, - self.owner_token) - self.assertEqual( - response.status_code, - status.HTTP_201_CREATED) - self.assertEqual( - content_data.get('value'), - 'Douglas') - - def test_forbidden(self): - user, user_token = create_user_and_token() - - response, content_data = self.request( - { - 'repository_translated_example': self.translated.id, - 'start': 11, - 'end': 18, - 'entity': 'name', - }, - user_token) - self.assertEqual( - response.status_code, - status.HTTP_403_FORBIDDEN) - - -class RepositoryTranslatedExampleEntityRetrieveTestCase(TestCase): - def setUp(self): - self.factory = RequestFactory() - - self.owner, self.owner_token = create_user_and_token('owner') - - self.repository = Repository.objects.create( - owner=self.owner, - name='Testing', - slug='test', - language=languages.LANGUAGE_EN) - self.example = RepositoryExample.objects.create( - repository_update=self.repository.current_update(), - text='my name is Douglas') - self.translated = RepositoryTranslatedExample.objects.create( - original_example=self.example, - language=languages.LANGUAGE_PT, - text='meu nome é Douglas') - self.translated_entity = RepositoryTranslatedExampleEntity.objects \ - .create( - repository_translated_example=self.translated, - start=11, - end=18, - entity='name') - - self.private_repository = Repository.objects.create( - owner=self.owner, - name='Private', - slug='private', - language=languages.LANGUAGE_EN, - is_private=True) - self.private_example = RepositoryExample.objects.create( - repository_update=self.private_repository.current_update(), - text='my name is Douglas') - self.private_translated = RepositoryTranslatedExample.objects.create( - original_example=self.private_example, - language=languages.LANGUAGE_PT, - text='meu nome é Douglas') - self.private_translated_entity = RepositoryTranslatedExampleEntity \ - .objects.create( - repository_translated_example=self.private_translated, - start=11, - end=18, - entity='name') - - def request(self, translated_entity, token): - authorization_header = { - 'HTTP_AUTHORIZATION': 'Token {}'.format(token.key), - } - request = self.factory.get( - '/api/translation-entity/{}/'.format(translated_entity.id), - **authorization_header) - response = RepositoryTranslatedExampleEntityViewSet.as_view( - {'get': 'retrieve'})(request, pk=translated_entity.id) - response.render() - content_data = json.loads(response.content) - return (response, content_data,) - - def test_okay(self): - response, content_data = self.request( - self.translated_entity, - self.owner_token) - self.assertEqual( - response.status_code, - status.HTTP_200_OK) - self.assertEqual( - content_data.get('id'), - self.translated_entity.id) - - def test_private_okay(self): - response, content_data = self.request( - self.private_translated_entity, - self.owner_token) - self.assertEqual( - response.status_code, - status.HTTP_200_OK) - self.assertEqual( - content_data.get('id'), - self.private_translated_entity.id) - - def test_forbidden(self): - user, user_token = create_user_and_token() - - response, content_data = self.request( - self.private_translated_entity, - user_token) - self.assertEqual( - response.status_code, - status.HTTP_403_FORBIDDEN) - - -class RepositoryTranslatedExampleEntityDestroyTestCase(TestCase): - def setUp(self): - self.factory = RequestFactory() - - self.owner, self.owner_token = create_user_and_token('owner') - - self.repository = Repository.objects.create( - owner=self.owner, - name='Testing', - slug='test', - language=languages.LANGUAGE_EN) - self.example = RepositoryExample.objects.create( - repository_update=self.repository.current_update(), - text='my name is Douglas') - self.translated = RepositoryTranslatedExample.objects.create( - original_example=self.example, - language=languages.LANGUAGE_PT, - text='meu nome é Douglas') - self.translated_entity = RepositoryTranslatedExampleEntity.objects \ - .create( - repository_translated_example=self.translated, - start=11, - end=18, - entity='name') - - def request(self, translated_entity, token): - authorization_header = { - 'HTTP_AUTHORIZATION': 'Token {}'.format(token.key), - } - request = self.factory.delete( - '/api/translation-entity/{}/'.format(translated_entity.id), - **authorization_header) - response = RepositoryTranslatedExampleEntityViewSet.as_view( - {'delete': 'destroy'})(request, pk=translated_entity.id) - return response - - def test_okay(self): - response = self.request( - self.translated, - self.owner_token) - self.assertEqual( - response.status_code, - status.HTTP_204_NO_CONTENT) - - def test_forbidden(self): - user, user_token = create_user_and_token() - - response = self.request( - self.translated, - user_token) - self.assertEqual( - response.status_code, - status.HTTP_403_FORBIDDEN) - - class TranslationsViewTest(TestCase): def setUp(self): self.factory = RequestFactory() diff --git a/bothub/api/validators.py b/bothub/api/validators.py index 919d28550..125ed61af 100644 --- a/bothub/api/validators.py +++ b/bothub/api/validators.py @@ -82,3 +82,13 @@ def __call__(self, attrs): if not intent and not entities: raise ValidationError(_('Define a intent or one entity')) + + +class EntityNotEqualLabelValidator(object): + def __call__(self, attrs): + entity = attrs.get('entity') + label = attrs.get('label') + + if entity == label: + raise ValidationError({'label': _( + 'Label name can\'t be equal to entity name')}) diff --git a/bothub/api/views.py b/bothub/api/views.py index cff7ab2fd..fd437c12a 100644 --- a/bothub/api/views.py +++ b/bothub/api/views.py @@ -23,11 +23,11 @@ from bothub.common.models import Repository from bothub.common.models import RepositoryExample from bothub.common.models import RepositoryTranslatedExample -from bothub.common.models import RepositoryTranslatedExampleEntity from bothub.common.models import RepositoryCategory from bothub.common.models import RepositoryVote from bothub.common.models import RepositoryAuthorization from bothub.common.models import RequestRepositoryAuthorization +from bothub.common.models import RepositoryEntity from bothub.authentication.models import User from .serializers import RepositorySerializer @@ -35,7 +35,6 @@ from .serializers import RepositoryExampleSerializer from .serializers import RepositoryAuthorizationSerializer from .serializers import RepositoryTranslatedExampleSerializer -from .serializers import RepositoryTranslatedExampleEntitySeralizer from .serializers import RegisterUserSerializer from .serializers import UserSerializer from .serializers import ChangePasswordSerializer @@ -52,6 +51,7 @@ from .serializers import NewRequestRepositoryAuthorizationSerializer from .serializers import RequestRepositoryAuthorizationSerializer from .serializers import ReviewAuthorizationRequestSerializer +from .serializers import RepositoryEntitySerializer # Permisions @@ -107,6 +107,18 @@ def has_object_permission(self, request, view, obj): return authorization.is_admin +class RepositoryEntityHasPermission(permissions.BasePermission): + def has_object_permission(self, request, view, obj): + authorization = obj.repository.get_user_authorization(request.user) + if request.method in READ_METHODS: + return authorization.can_read + if request.user.is_authenticated: + if request.method in WRITE_METHODS: + return authorization.can_write + return authorization.is_admin + return False + + # Filters class ExamplesFilter(filters.FilterSet): @@ -286,6 +298,35 @@ def filter_repository_uuid(self, queryset, name, value): raise NotFound(_('Invalid repository UUID')) +class RepositoryEntitiesFilter(filters.FilterSet): + class Meta: + model = RepositoryEntity + fields = [ + 'repository_uuid', + 'value', + ] + + repository_uuid = filters.CharFilter( + name='repository_uuid', + required=True, + method='filter_repository_uuid', + help_text=_('Repository\'s UUID')) + + def filter_repository_uuid(self, queryset, name, value): + request = self.request + try: + repository = Repository.objects.get(uuid=value) + authorization = repository.get_user_authorization(request.user) + if not authorization.is_admin: + raise PermissionDenied() + return queryset.filter(repository=repository) + except Repository.DoesNotExist: + raise NotFound( + _('Repository {} does not exist').format(value)) + except DjangoValidationError: + raise NotFound(_('Invalid repository UUID')) + + # Mixins class MultipleFieldLookupMixin(object): @@ -580,38 +621,6 @@ class RepositoryTranslatedExampleViewSet( ] -class NewRepositoryTranslatedExampleEntityViewSet( - mixins.CreateModelMixin, - GenericViewSet): - """ - Add entity to example translation - """ - queryset = RepositoryTranslatedExampleEntity.objects - serializer_class = RepositoryTranslatedExampleEntitySeralizer - permission_classes = [permissions.IsAuthenticated] - - -class RepositoryTranslatedExampleEntityViewSet( - mixins.RetrieveModelMixin, - mixins.DestroyModelMixin, - GenericViewSet): - """ - Manage translation entity - - retrieve: - Get translation entity data. - - delete: - Delete translation entity. - """ - queryset = RepositoryTranslatedExampleEntity.objects - serializer_class = RepositoryTranslatedExampleEntitySeralizer - permission_classes = [ - permissions.IsAuthenticated, - RepositoryTranslatedExampleEntityPermission, - ] - - class RepositoryExamplesViewSet( mixins.ListModelMixin, GenericViewSet): @@ -930,3 +939,15 @@ def update(self, *args, **kwargs): return super().update(*args, **kwargs) except DjangoValidationError as e: raise ValidationError(e.message) + + +class RepositoryEntitiesViewSet( + mixins.ListModelMixin, + GenericViewSet): + queryset = RepositoryEntity.objects.all() + serializer_class = RepositoryEntitySerializer + filter_class = RepositoryEntitiesFilter + permission_classes = [ + IsAuthenticated, + RepositoryEntityHasPermission, + ] diff --git a/bothub/common/migrations/0018_auto_20180725_1305.py b/bothub/common/migrations/0018_auto_20180725_1305.py new file mode 100644 index 000000000..162767830 --- /dev/null +++ b/bothub/common/migrations/0018_auto_20180725_1305.py @@ -0,0 +1,59 @@ +# Generated by Django 2.0.6 on 2018-07-25 13:05 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import re + + +def populate_example_entities(apps, *args): + RepositoryExampleEntity = apps.get_model('common', 'RepositoryExampleEntity') + RepositoryTranslatedExampleEntity = apps.get_model('common', 'RepositoryTranslatedExampleEntity') + RepositoryEntity = apps.get_model('common', 'RepositoryEntity') + + for e in RepositoryExampleEntity.objects.all(): + entity, create = RepositoryEntity.objects.get_or_create( + repository=e.repository_example.repository_update.repository, + value=e.entity) + e.entity = entity.pk + e.save() + + for e in RepositoryTranslatedExampleEntity.objects.all(): + entity, create = RepositoryEntity.objects.get_or_create( + repository=e.repository_translated_example.repository_update.repository, + value=e.entity) + e.entity = entity.pk + e.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('common', '0017_auto_20180712_2131'), + ] + + operations = [ + migrations.CreateModel( + name='RepositoryEntity', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('value', models.CharField(help_text='Entity name', max_length=64, validators=[django.core.validators.RegexValidator(re.compile('^[-a-z0-9_]+\\Z'), 'Enter a valid value consisting of lowercase letters, numbers, underscores or hyphens.', 'invalid')], verbose_name='entity')), + ('repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='entities', to='common.Repository')), + ], + ), + migrations.RunPython(populate_example_entities), + migrations.AlterField( + model_name='repositoryexampleentity', + name='entity', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='common.RepositoryEntity'), + ), + migrations.AlterField( + model_name='repositorytranslatedexampleentity', + name='entity', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='common.RepositoryEntity'), + ), + migrations.AlterUniqueTogether( + name='repositoryentity', + unique_together={('repository', 'value')}, + ), + ] diff --git a/bothub/common/migrations/0019_auto_20180725_1657.py b/bothub/common/migrations/0019_auto_20180725_1657.py new file mode 100644 index 000000000..f5a64ccc3 --- /dev/null +++ b/bothub/common/migrations/0019_auto_20180725_1657.py @@ -0,0 +1,41 @@ +# Generated by Django 2.0.6 on 2018-07-25 16:57 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import re + + +class Migration(migrations.Migration): + + dependencies = [ + ('common', '0018_auto_20180725_1305'), + ] + + operations = [ + migrations.CreateModel( + name='RepositoryEntityLabel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('value', models.CharField(blank=True, max_length=64, validators=[django.core.validators.RegexValidator(re.compile('^[-a-z0-9_]+\\Z'), 'Enter a valid value consisting of lowercase letters, numbers, underscores or hyphens.', 'invalid')], verbose_name='label')), + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='created at')), + ('repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='labels', to='common.Repository')), + ], + ), + migrations.AddField( + model_name='repositoryentity', + name='created_at', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now, verbose_name='created at'), + preserve_default=False, + ), + migrations.AddField( + model_name='repositoryentity', + name='label', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='entities', to='common.RepositoryEntityLabel'), + ), + migrations.AlterUniqueTogether( + name='repositoryentitylabel', + unique_together={('repository', 'value')}, + ), + ] diff --git a/bothub/common/migrations/0020_auto_20180813_1320.py b/bothub/common/migrations/0020_auto_20180813_1320.py new file mode 100644 index 000000000..ebe2eb388 --- /dev/null +++ b/bothub/common/migrations/0020_auto_20180813_1320.py @@ -0,0 +1,21 @@ +# Generated by Django 2.0.6 on 2018-08-13 13:20 + +import bothub.common.models +import django.core.validators +from django.db import migrations, models +import re + + +class Migration(migrations.Migration): + + dependencies = [ + ('common', '0019_auto_20180725_1657'), + ] + + operations = [ + migrations.AlterField( + model_name='repositoryentitylabel', + name='value', + field=models.CharField(blank=True, max_length=64, validators=[django.core.validators.RegexValidator(re.compile('^[-a-z0-9_]+\\Z'), 'Enter a valid value consisting of lowercase letters, numbers, underscores or hyphens.', 'invalid'), bothub.common.models.can_t_be_other], verbose_name='label'), + ), + ] diff --git a/bothub/common/models.py b/bothub/common/models.py index 816cf420e..639c20f78 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -21,15 +21,20 @@ from .exceptions import DoesNotHaveTranslation -entity_and_intent_regex = _lazy_re_compile(r'^[-a-z0-9_]+\Z') -validate_entity_and_intent = RegexValidator( - entity_and_intent_regex, +item_key_regex = _lazy_re_compile(r'^[-a-z0-9_]+\Z') +validate_item_key = RegexValidator( + item_key_regex, _('Enter a valid value consisting of lowercase letters, numbers, ' + 'underscores or hyphens.'), 'invalid' ) +def can_t_be_other(value): + if value == 'other': + raise ValidationError(_('The label can\'t be named as "other"')) + + class RepositoryCategory(models.Model): class Meta: verbose_name = _('repository category') @@ -191,11 +196,8 @@ def intents(self): flat=True))) @property - def entities(self): - return list(set(self.examples().annotate( - entities_count=models.Count( - 'entities')).filter(entities_count__gte=1).values_list( - 'entities__entity', flat=True))) + def entities_list(self): + return list(set(self.entities.all().values_list('value', flat=True))) @property def admins(self): @@ -410,7 +412,7 @@ class Meta: max_length=64, blank=True, help_text=_('Example intent reference'), - validators=[validate_entity_and_intent]) + validators=[validate_item_key]) created_at = models.DateTimeField( _('created at'), auto_now_add=True) @@ -440,15 +442,6 @@ def get_entities(self, language): return self.entities.all() return self.get_translation(language).entities.all() - def rasa_nlu_data(self, language): - return { - 'text': self.get_text(language), - 'intent': self.intent, - 'entities': [ - entity.rasa_nlu_data for entity in self.get_entities( - language)], - } - def delete(self): self.deleted_in = self.repository_update.repository.current_update( self.repository_update.language) @@ -539,6 +532,115 @@ def has_valid_entities(self): list(map(lambda x: x.to_dict, my_entities))) +class RepositoryEntityLabelQueryset(models.QuerySet): + def get(self, repository, value): + try: + return super().get( + repository=repository, + value=value) + except self.model.DoesNotExist as e: + return super().create( + repository=repository, + value=value) + + +class RepositoryEntityLabelManager(models.Manager): + def get_queryset(self): + return RepositoryEntityLabelQueryset(self.model, using=self._db) + + +class RepositoryEntityLabel(models.Model): + class Meta: + unique_together = ['repository', 'value'] + + repository = models.ForeignKey( + Repository, + on_delete=models.CASCADE, + related_name='labels') + value = models.CharField( + _('label'), + max_length=64, + validators=[ + validate_item_key, + can_t_be_other, + ], + blank=True) + created_at = models.DateTimeField( + _('created at'), + auto_now_add=True) + + objects = RepositoryEntityLabelManager() + + +class RepositoryEntityQueryset(models.QuerySet): + def get(self, repository, value): + try: + return super().get( + repository=repository, + value=value) + except self.model.DoesNotExist as e: + return super().create( + repository=repository, + value=value) + + +class RepositoryEntityManager(models.Manager): + def get_queryset(self): + return RepositoryEntityQueryset(self.model, using=self._db) + + +class RepositoryEntity(models.Model): + class Meta: + unique_together = ['repository', 'value'] + + repository = models.ForeignKey( + Repository, + on_delete=models.CASCADE, + related_name='entities') + value = models.CharField( + _('entity'), + max_length=64, + help_text=_('Entity name'), + validators=[validate_item_key]) + label = models.ForeignKey( + RepositoryEntityLabel, + on_delete=models.CASCADE, + related_name='entities', + null=True, + blank=True) + created_at = models.DateTimeField( + _('created at'), + auto_now_add=True) + + objects = RepositoryEntityManager() + + def set_label(self, value): + if not value: + self.label = None + else: + self.label = RepositoryEntityLabel.objects.get( + repository=self.repository, + value=value) + + +class EntityBaseQueryset(models.QuerySet): + def create(self, entity, **kwargs): + if type(entity) is not RepositoryEntity: + instance = self.model(**kwargs) + repository = instance.example.repository_update.repository + entity = RepositoryEntity.objects.get( + repository=repository, + value=entity) + return super().create( + entity=entity, + **kwargs) + + +class EntityBaseManager(models.Manager): + def get_queryset(self): + return EntityBaseQueryset(self.model, using=self._db) + + class EntityBase(models.Model): class Meta: verbose_name = _('repository example entity') @@ -551,18 +653,22 @@ class Meta: end = models.PositiveIntegerField( _('end'), help_text=_('End index of entity value in example text')) - entity = models.CharField( - _('entity'), - max_length=64, - help_text=_('Entity name'), - validators=[validate_entity_and_intent]) + entity = models.ForeignKey( + RepositoryEntity, + on_delete=models.CASCADE) created_at = models.DateTimeField( _('created at'), auto_now_add=True) + objects = EntityBaseManager() + + @property + def example(self): + return self.get_example() + @property def value(self): - return self.get_example().text[self.start:self.end] + return self.example.text[self.start:self.end] @property def rasa_nlu_data(self): @@ -570,20 +676,24 @@ def rasa_nlu_data(self): 'start': self.start, 'end': self.end, 'value': self.value, - 'entity': self.entity, + 'entity': self.entity.value, } @property def to_dict(self): + return self.get_rasa_nlu_data() + + def get_example(self): + pass # pragma: no cover + + def get_rasa_nlu_data(self, label_as_entity=False): return { 'start': self.start, 'end': self.end, - 'entity': self.entity, + 'entity': self.entity.label.value + if label_as_entity else self.entity.value, } - def get_example(self): - pass # pragma: no cover - class RepositoryExampleEntity(EntityBase): repository_example = models.ForeignKey( diff --git a/bothub/common/tests.py b/bothub/common/tests.py index cb3a07e4e..63cb95d4b 100644 --- a/bothub/common/tests.py +++ b/bothub/common/tests.py @@ -11,12 +11,14 @@ from .models import RepositoryTranslatedExample from .models import RepositoryTranslatedExampleEntity from .models import RepositoryAuthorization -from .models import DoesNotHaveTranslation from .models import RequestRepositoryAuthorization +from .models import RepositoryEntity +from .models import RepositoryEntityLabel from . import languages from .exceptions import RepositoryUpdateAlreadyStartedTraining from .exceptions import RepositoryUpdateAlreadyTrained from .exceptions import TrainingNotAllowed +from .exceptions import DoesNotHaveTranslation class RepositoryUpdateTestCase(TestCase): @@ -88,17 +90,6 @@ def test_new_translate(self): len(self.repository.current_update(language).examples), 1) - def test_to_rasa_nlu_data(self): - language = languages.LANGUAGE_PT - RepositoryTranslatedExample.objects.create( - original_example=self.example, - language=language, - text='meu nome é Douglas') - - self.assertDictEqual( - self.example.rasa_nlu_data(language), - TranslateTestCase.EXPECTED_RASA_NLU_DATA) - def test_translated_entity(self): RepositoryExampleEntity.objects.create( repository_example=self.example, @@ -116,9 +107,6 @@ def test_translated_entity(self): start=11, end=18, entity='name') - self.assertDictEqual( - self.example.rasa_nlu_data(language), - TranslateTestCase.EXPECTED_RASA_NLU_DATA_WITH_ENTITIES) def test_valid_entities(self): RepositoryExampleEntity.objects.create( @@ -330,7 +318,7 @@ def test_entities(self): self.assertIn( 'name', - self.repository.entities) + self.repository.entities.values_list('value', flat=True)) def test_not_blank_value_in_intents(self): RepositoryExample.objects.create( @@ -810,3 +798,142 @@ def test_approve_twice_another_admin(self): with self.assertRaises(ValidationError): self.ra.approved_by = self.admin self.ra.save() + + +class RepositoryEntityTestCase(TestCase): + def setUp(self): + self.language = languages.LANGUAGE_EN + + self.owner = User.objects.create_user('owner@user.com', 'user') + + self.repository = Repository.objects.create( + owner=self.owner, + name='Test', + slug='test', + language=self.language) + + self.example = RepositoryExample.objects.create( + repository_update=self.repository.current_update(), + text='my name is Douglas') + + self.example_entity_1 = RepositoryExampleEntity.objects.create( + repository_example=self.example, + start=11, + end=18, + entity='name') + + self.example_entity_2 = RepositoryExampleEntity.objects.create( + repository_example=self.example, + start=0, + end=2, + entity='object') + + def test_example_entity_create_entity(self): + name_entity = RepositoryEntity.objects.get( + repository=self.repository, + value='name') + self.assertEqual( + name_entity.pk, + self.example_entity_1.pk) + + def test_dont_duplicate_entity(self): + name_entity = RepositoryEntity.objects.get( + repository=self.repository, + value='name') + + new_example_entity = RepositoryExampleEntity.objects.create( + repository_example=self.example, + start=11, + end=18, + entity='name') + + self.assertEqual( + name_entity.pk, + self.example_entity_1.pk) + self.assertEqual( + name_entity.pk, + new_example_entity.entity.pk) + + +class RepositoryEntityLabelTestCase(TestCase): + def setUp(self): + self.language = languages.LANGUAGE_EN + + self.owner = User.objects.create_user('owner@user.com', 'user') + + self.repository = Repository.objects.create( + owner=self.owner, + name='Test', + slug='test', + language=self.language) + + self.example = RepositoryExample.objects.create( + repository_update=self.repository.current_update(), + text='my name is Douglas') + + self.example_entity_1 = RepositoryExampleEntity.objects.create( + repository_example=self.example, + start=11, + end=18, + entity='name') + + self.example_entity_2 = RepositoryExampleEntity.objects.create( + repository_example=self.example, + start=0, + end=2, + entity='object') + + def test_set_label(self): + name_entity = RepositoryEntity.objects.get( + repository=self.repository, + value='name') + + name_entity.set_label('subject') + + self.assertIsNotNone(name_entity.label) + + def test_entity_label_created(self): + name_entity = RepositoryEntity.objects.get( + repository=self.repository, + value='name') + + name_entity.set_label('subject') + + subject_label = RepositoryEntityLabel.objects.get( + repository=self.repository, + value='subject') + + self.assertEqual( + name_entity.label.pk, + subject_label.pk) + + def test_dont_duplicate_label(self): + name_entity = RepositoryEntity.objects.get( + repository=self.repository, + value='name') + name_entity.set_label('subject') + + object_entity = RepositoryEntity.objects.get( + repository=self.repository, + value='object') + object_entity.set_label('subject') + + subject_label = RepositoryEntityLabel.objects.get( + repository=self.repository, + value='subject') + + self.assertEqual( + name_entity.label.pk, + subject_label.pk) + self.assertEqual( + object_entity.label.pk, + subject_label.pk) + + def test_set_label_to_none(self): + name_entity = RepositoryEntity.objects.get( + repository=self.repository, + value='name') + + name_entity.set_label(None) + + self.assertIsNone(name_entity.label)