From 6068d2c87d27f4d7fc3a0914a41fab096e3907e2 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Wed, 25 Jul 2018 11:34:58 -0300 Subject: [PATCH 01/19] Add RepositoryEntity model --- .../migrations/0018_auto_20180725_1305.py | 59 +++++++++++++++ bothub/common/models.py | 72 ++++++++++++++++--- bothub/common/tests.py | 2 +- 3 files changed, 124 insertions(+), 9 deletions(-) create mode 100644 bothub/common/migrations/0018_auto_20180725_1305.py diff --git a/bothub/common/migrations/0018_auto_20180725_1305.py b/bothub/common/migrations/0018_auto_20180725_1305.py new file mode 100644 index 000000000..162767830 --- /dev/null +++ b/bothub/common/migrations/0018_auto_20180725_1305.py @@ -0,0 +1,59 @@ +# Generated by Django 2.0.6 on 2018-07-25 13:05 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import re + + +def populate_example_entities(apps, *args): + RepositoryExampleEntity = apps.get_model('common', 'RepositoryExampleEntity') + RepositoryTranslatedExampleEntity = apps.get_model('common', 'RepositoryTranslatedExampleEntity') + RepositoryEntity = apps.get_model('common', 'RepositoryEntity') + + for e in RepositoryExampleEntity.objects.all(): + entity, create = RepositoryEntity.objects.get_or_create( + repository=e.repository_example.repository_update.repository, + value=e.entity) + e.entity = entity.pk + e.save() + + for e in RepositoryTranslatedExampleEntity.objects.all(): + entity, create = RepositoryEntity.objects.get_or_create( + repository=e.repository_translated_example.repository_update.repository, + value=e.entity) + e.entity = entity.pk + e.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('common', '0017_auto_20180712_2131'), + ] + + operations = [ + migrations.CreateModel( + name='RepositoryEntity', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('value', models.CharField(help_text='Entity name', max_length=64, validators=[django.core.validators.RegexValidator(re.compile('^[-a-z0-9_]+\\Z'), 'Enter a valid value consisting of lowercase letters, numbers, underscores or hyphens.', 'invalid')], verbose_name='entity')), + ('repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='entities', to='common.Repository')), + ], + ), + migrations.RunPython(populate_example_entities), + migrations.AlterField( + model_name='repositoryexampleentity', + name='entity', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='common.RepositoryEntity'), + ), + migrations.AlterField( + model_name='repositorytranslatedexampleentity', + name='entity', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='common.RepositoryEntity'), + ), + migrations.AlterUniqueTogether( + name='repositoryentity', + unique_together={('repository', 'value')}, + ), + ] diff --git a/bothub/common/models.py b/bothub/common/models.py index c5f85ac61..be81a3f9b 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -545,6 +545,58 @@ def has_valid_entities(self): list(map(lambda x: x.to_dict, my_entities))) +class RepositoryEntityQueryset(models.QuerySet): + def get(self, repository, value): + try: + return super().get( + repository=repository, + value=value) + except self.model.DoesNotExist as e: + return super().create( + repository=repository, + value=value) + + +class RepositoryEntityManager(models.Manager): + def get_queryset(self): + return RepositoryEntityQueryset(self.model, using=self._db) + + +class RepositoryEntity(models.Model): + class Meta: + unique_together = ['repository', 'value'] + + repository = models.ForeignKey( + Repository, + on_delete=models.CASCADE, + related_name='entities') + value = models.CharField( + _('entity'), + max_length=64, + help_text=_('Entity name'), + validators=[validate_entity_and_intent]) + + objects = RepositoryEntityManager() + + +class EntityBaseQueryset(models.QuerySet): + def create(self, entity, **kwargs): + if type(entity) is not RepositoryEntity: + instance = self.model(**kwargs) + repository = instance.example.repository_update.repository + entity = RepositoryEntity.objects.get( + repository=repository, + value=entity) + return super().create( + entity=entity, + **kwargs) + + +class EntityBaseManager(models.Manager): + def get_queryset(self): + return EntityBaseQueryset(self.model, using=self._db) + + class EntityBase(models.Model): class Meta: verbose_name = _('repository example entity') @@ -557,18 +609,22 @@ class Meta: end = models.PositiveIntegerField( _('end'), help_text=_('End index of entity value in example text')) - entity = models.CharField( - _('entity'), - max_length=64, - help_text=_('Entity name'), - validators=[validate_entity_and_intent]) + entity = models.ForeignKey( + RepositoryEntity, + on_delete=models.CASCADE) created_at = models.DateTimeField( _('created at'), auto_now_add=True) + objects = EntityBaseManager() + + @property + def example(self): + return self.get_example() + @property def value(self): - return self.get_example().text[self.start:self.end] + return self.example.text[self.start:self.end] @property def rasa_nlu_data(self): @@ -576,7 +632,7 @@ def rasa_nlu_data(self): 'start': self.start, 'end': self.end, 'value': self.value, - 'entity': self.entity, + 'entity': self.entity.value, } @property @@ -584,7 +640,7 @@ def to_dict(self): return { 'start': self.start, 'end': self.end, - 'entity': self.entity, + 'entity': self.entity.value, } def get_example(self): diff --git a/bothub/common/tests.py b/bothub/common/tests.py index 7c7c57890..df29efed8 100644 --- a/bothub/common/tests.py +++ b/bothub/common/tests.py @@ -314,7 +314,7 @@ def test_entities(self): self.assertIn( 'name', - self.repository.entities) + self.repository.entities.values_list('value', flat=True)) def test_not_blank_value_in_intents(self): RepositoryExample.objects.create( From ad31ed4d9be3b02013c448997b92670d1e113e42 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Wed, 25 Jul 2018 13:39:01 -0300 Subject: [PATCH 02/19] Resolve property conflict in Repository model, rename entities -> entities_list --- bothub/common/models.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/bothub/common/models.py b/bothub/common/models.py index be81a3f9b..3091398dd 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -193,11 +193,8 @@ def intents(self): flat=True))) @property - def entities(self): - return list(set(self.examples().annotate( - entities_count=models.Count( - 'entities')).filter(entities_count__gte=1).values_list( - 'entities__entity', flat=True))) + def entities_list(self): + return list(set(self.entities.all().values_list('value', flat=True))) @property def admins(self): From 43dd968f7ab5a2749cdcbe93871e5c203bc5d0e1 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Wed, 25 Jul 2018 13:40:47 -0300 Subject: [PATCH 03/19] Prepare api to new entity format --- bothub/api/fields.py | 19 +++ bothub/api/routers.py | 6 - bothub/api/serializers/example.py | 7 + bothub/api/serializers/repository.py | 4 + bothub/api/serializers/translate.py | 7 + bothub/api/tests/test_translate.py | 214 --------------------------- bothub/api/views.py | 34 ----- 7 files changed, 37 insertions(+), 254 deletions(-) diff --git a/bothub/api/fields.py b/bothub/api/fields.py index e0ce136a3..79e7f616f 100644 --- a/bothub/api/fields.py +++ b/bothub/api/fields.py @@ -1,5 +1,7 @@ from rest_framework import serializers +from bothub.common.models import RepositoryEntity + class ModelMultipleChoiceField(serializers.ManyRelatedField): pass @@ -17,3 +19,20 @@ def __init__(self, *args, **kwargs): class EntityText(serializers.CharField): pass + + +class EntityValueField(serializers.CharField): + def __init__(self, *args, validators=[], **kwargs): + kwargs.pop('max_length', 0) + kwargs.pop('help_text', '') + + value_field = RepositoryEntity._meta.get_field('value') + + super().__init__( + *args, + max_length=value_field.max_length, + validators=(validators + value_field.validators), + **kwargs) + + def to_representation(self, obj): + return obj.value diff --git a/bothub/api/routers.py b/bothub/api/routers.py index 853655a8b..f6387e9d0 100644 --- a/bothub/api/routers.py +++ b/bothub/api/routers.py @@ -7,8 +7,6 @@ from .views import RepositoryExampleViewSet from .views import NewRepositoryTranslatedExampleViewSet from .views import RepositoryTranslatedExampleViewSet -from .views import NewRepositoryTranslatedExampleEntityViewSet -from .views import RepositoryTranslatedExampleEntityViewSet from .views import RepositoryExamplesViewSet from .views import RegisterUserViewSet from .views import LoginViewSet @@ -101,10 +99,6 @@ def get_lookup_regex(self, viewset, lookup_prefix=''): router.register('example', RepositoryExampleViewSet) router.register('translate-example', NewRepositoryTranslatedExampleViewSet) router.register('translation', RepositoryTranslatedExampleViewSet) -router.register('translation-entity/new', - NewRepositoryTranslatedExampleEntityViewSet) -router.register('translation-entity', - RepositoryTranslatedExampleEntityViewSet) router.register('examples', RepositoryExamplesViewSet) router.register('register', RegisterUserViewSet) router.register('login', LoginViewSet) diff --git a/bothub/api/serializers/example.py b/bothub/api/serializers/example.py index c6086fdc2..f97542554 100644 --- a/bothub/api/serializers/example.py +++ b/bothub/api/serializers/example.py @@ -7,6 +7,7 @@ from bothub.common.models import RepositoryExampleEntity from ..fields import EntityText +from ..fields import EntityValueField from ..validators import CanContributeInRepositoryExampleValidator from ..validators import CanContributeInRepositoryValidator from ..validators import ExampleWithIntentOrEntityValidator @@ -32,8 +33,12 @@ class Meta: CanContributeInRepositoryExampleValidator(), ], help_text=_('Example\'s ID')) + entity = serializers.SerializerMethodField() value = serializers.SerializerMethodField() + def get_entity(self, obj): + return obj.entity.value + def get_value(self, obj): return obj.value @@ -48,6 +53,8 @@ class Meta: 'entity', ] + entity = EntityValueField() + class RepositoryExampleSerializer(serializers.ModelSerializer): class Meta: diff --git a/bothub/api/serializers/repository.py b/bothub/api/serializers/repository.py index acfe33271..7120f1f19 100644 --- a/bothub/api/serializers/repository.py +++ b/bothub/api/serializers/repository.py @@ -87,6 +87,7 @@ class Meta: slug_field='nickname', read_only=True) categories_list = serializers.SerializerMethodField() + entities = serializers.SerializerMethodField() authorization = serializers.SerializerMethodField() examples__count = serializers.SerializerMethodField() request_authorization = serializers.SerializerMethodField() @@ -95,6 +96,9 @@ class Meta: def get_categories_list(self, obj): return RepositoryCategorySerializer(obj.categories, many=True).data + def get_entities(self, obj): + return obj.entities_list + def get_authorization(self, obj): request = self.context.get('request') if not request: diff --git a/bothub/api/serializers/translate.py b/bothub/api/serializers/translate.py index 4eab2fc3d..01fa9ce01 100644 --- a/bothub/api/serializers/translate.py +++ b/bothub/api/serializers/translate.py @@ -7,6 +7,7 @@ from bothub.common.models import RepositoryExample from bothub.common.languages import LANGUAGE_CHOICES +from ..fields import EntityValueField from ..validators import CanContributeInRepositoryTranslatedExampleValidator from ..validators import CanContributeInRepositoryExampleValidator from ..validators import TranslatedExampleEntitiesValidator @@ -32,8 +33,12 @@ class Meta: CanContributeInRepositoryTranslatedExampleValidator(), ], help_text='Example translation ID') + entity = serializers.SerializerMethodField() value = serializers.SerializerMethodField() + def get_entity(self, obj): + return obj.entity.value + def get_value(self, obj): return obj.value @@ -81,6 +86,8 @@ class Meta: 'entity', ] + entity = EntityValueField() + class NewRepositoryTranslatedExampleSerializer(serializers.ModelSerializer): class Meta: diff --git a/bothub/api/tests/test_translate.py b/bothub/api/tests/test_translate.py index 346241108..6dbd9a6ca 100644 --- a/bothub/api/tests/test_translate.py +++ b/bothub/api/tests/test_translate.py @@ -9,13 +9,10 @@ from bothub.common.models import Repository from bothub.common.models import RepositoryExample from bothub.common.models import RepositoryTranslatedExample -from bothub.common.models import RepositoryTranslatedExampleEntity from bothub.common.models import RepositoryExampleEntity from ..views import NewRepositoryTranslatedExampleViewSet from ..views import RepositoryTranslatedExampleViewSet -from ..views import NewRepositoryTranslatedExampleEntityViewSet -from ..views import RepositoryTranslatedExampleEntityViewSet from ..views import TranslationsViewSet from .utils import create_user_and_token @@ -308,217 +305,6 @@ def test_forbidden(self): status.HTTP_403_FORBIDDEN) -class NewRepositoryTranslatedExampleEntityTestCase(TestCase): - def setUp(self): - self.factory = RequestFactory() - - self.owner, self.owner_token = create_user_and_token('owner') - - self.repository = Repository.objects.create( - owner=self.owner, - name='Testing', - slug='test', - language=languages.LANGUAGE_EN) - self.example = RepositoryExample.objects.create( - repository_update=self.repository.current_update(), - text='my name is Douglas') - self.translated = RepositoryTranslatedExample.objects.create( - original_example=self.example, - language=languages.LANGUAGE_PT, - text='meu nome é Douglas') - - def request(self, data, token): - authorization_header = { - 'HTTP_AUTHORIZATION': 'Token {}'.format(token.key), - } - request = self.factory.post( - '/api/translate-example/', - data, - **authorization_header) - response = NewRepositoryTranslatedExampleEntityViewSet.as_view( - {'post': 'create'})(request) - response.render() - content_data = json.loads(response.content) - return (response, content_data,) - - def test_okay(self): - response, content_data = self.request( - { - 'repository_translated_example': self.translated.id, - 'start': 11, - 'end': 18, - 'entity': 'name', - }, - self.owner_token) - self.assertEqual( - response.status_code, - status.HTTP_201_CREATED) - self.assertEqual( - content_data.get('value'), - 'Douglas') - - def test_forbidden(self): - user, user_token = create_user_and_token() - - response, content_data = self.request( - { - 'repository_translated_example': self.translated.id, - 'start': 11, - 'end': 18, - 'entity': 'name', - }, - user_token) - self.assertEqual( - response.status_code, - status.HTTP_403_FORBIDDEN) - - -class RepositoryTranslatedExampleEntityRetrieveTestCase(TestCase): - def setUp(self): - self.factory = RequestFactory() - - self.owner, self.owner_token = create_user_and_token('owner') - - self.repository = Repository.objects.create( - owner=self.owner, - name='Testing', - slug='test', - language=languages.LANGUAGE_EN) - self.example = RepositoryExample.objects.create( - repository_update=self.repository.current_update(), - text='my name is Douglas') - self.translated = RepositoryTranslatedExample.objects.create( - original_example=self.example, - language=languages.LANGUAGE_PT, - text='meu nome é Douglas') - self.translated_entity = RepositoryTranslatedExampleEntity.objects \ - .create( - repository_translated_example=self.translated, - start=11, - end=18, - entity='name') - - self.private_repository = Repository.objects.create( - owner=self.owner, - name='Private', - slug='private', - language=languages.LANGUAGE_EN, - is_private=True) - self.private_example = RepositoryExample.objects.create( - repository_update=self.private_repository.current_update(), - text='my name is Douglas') - self.private_translated = RepositoryTranslatedExample.objects.create( - original_example=self.private_example, - language=languages.LANGUAGE_PT, - text='meu nome é Douglas') - self.private_translated_entity = RepositoryTranslatedExampleEntity \ - .objects.create( - repository_translated_example=self.private_translated, - start=11, - end=18, - entity='name') - - def request(self, translated_entity, token): - authorization_header = { - 'HTTP_AUTHORIZATION': 'Token {}'.format(token.key), - } - request = self.factory.get( - '/api/translation-entity/{}/'.format(translated_entity.id), - **authorization_header) - response = RepositoryTranslatedExampleEntityViewSet.as_view( - {'get': 'retrieve'})(request, pk=translated_entity.id) - response.render() - content_data = json.loads(response.content) - return (response, content_data,) - - def test_okay(self): - response, content_data = self.request( - self.translated_entity, - self.owner_token) - self.assertEqual( - response.status_code, - status.HTTP_200_OK) - self.assertEqual( - content_data.get('id'), - self.translated_entity.id) - - def test_private_okay(self): - response, content_data = self.request( - self.private_translated_entity, - self.owner_token) - self.assertEqual( - response.status_code, - status.HTTP_200_OK) - self.assertEqual( - content_data.get('id'), - self.private_translated_entity.id) - - def test_forbidden(self): - user, user_token = create_user_and_token() - - response, content_data = self.request( - self.private_translated_entity, - user_token) - self.assertEqual( - response.status_code, - status.HTTP_403_FORBIDDEN) - - -class RepositoryTranslatedExampleEntityDestroyTestCase(TestCase): - def setUp(self): - self.factory = RequestFactory() - - self.owner, self.owner_token = create_user_and_token('owner') - - self.repository = Repository.objects.create( - owner=self.owner, - name='Testing', - slug='test', - language=languages.LANGUAGE_EN) - self.example = RepositoryExample.objects.create( - repository_update=self.repository.current_update(), - text='my name is Douglas') - self.translated = RepositoryTranslatedExample.objects.create( - original_example=self.example, - language=languages.LANGUAGE_PT, - text='meu nome é Douglas') - self.translated_entity = RepositoryTranslatedExampleEntity.objects \ - .create( - repository_translated_example=self.translated, - start=11, - end=18, - entity='name') - - def request(self, translated_entity, token): - authorization_header = { - 'HTTP_AUTHORIZATION': 'Token {}'.format(token.key), - } - request = self.factory.delete( - '/api/translation-entity/{}/'.format(translated_entity.id), - **authorization_header) - response = RepositoryTranslatedExampleEntityViewSet.as_view( - {'delete': 'destroy'})(request, pk=translated_entity.id) - return response - - def test_okay(self): - response = self.request( - self.translated, - self.owner_token) - self.assertEqual( - response.status_code, - status.HTTP_204_NO_CONTENT) - - def test_forbidden(self): - user, user_token = create_user_and_token() - - response = self.request( - self.translated, - user_token) - self.assertEqual( - response.status_code, - status.HTTP_403_FORBIDDEN) - - class TranslationsViewTest(TestCase): def setUp(self): self.factory = RequestFactory() diff --git a/bothub/api/views.py b/bothub/api/views.py index cff7ab2fd..7b6b89317 100644 --- a/bothub/api/views.py +++ b/bothub/api/views.py @@ -23,7 +23,6 @@ from bothub.common.models import Repository from bothub.common.models import RepositoryExample from bothub.common.models import RepositoryTranslatedExample -from bothub.common.models import RepositoryTranslatedExampleEntity from bothub.common.models import RepositoryCategory from bothub.common.models import RepositoryVote from bothub.common.models import RepositoryAuthorization @@ -35,7 +34,6 @@ from .serializers import RepositoryExampleSerializer from .serializers import RepositoryAuthorizationSerializer from .serializers import RepositoryTranslatedExampleSerializer -from .serializers import RepositoryTranslatedExampleEntitySeralizer from .serializers import RegisterUserSerializer from .serializers import UserSerializer from .serializers import ChangePasswordSerializer @@ -580,38 +578,6 @@ class RepositoryTranslatedExampleViewSet( ] -class NewRepositoryTranslatedExampleEntityViewSet( - mixins.CreateModelMixin, - GenericViewSet): - """ - Add entity to example translation - """ - queryset = RepositoryTranslatedExampleEntity.objects - serializer_class = RepositoryTranslatedExampleEntitySeralizer - permission_classes = [permissions.IsAuthenticated] - - -class RepositoryTranslatedExampleEntityViewSet( - mixins.RetrieveModelMixin, - mixins.DestroyModelMixin, - GenericViewSet): - """ - Manage translation entity - - retrieve: - Get translation entity data. - - delete: - Delete translation entity. - """ - queryset = RepositoryTranslatedExampleEntity.objects - serializer_class = RepositoryTranslatedExampleEntitySeralizer - permission_classes = [ - permissions.IsAuthenticated, - RepositoryTranslatedExampleEntityPermission, - ] - - class RepositoryExamplesViewSet( mixins.ListModelMixin, GenericViewSet): From 836577e0b462286ed142b8e4108d175ef5a3f81d Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Wed, 25 Jul 2018 14:45:05 -0300 Subject: [PATCH 04/19] Add tests to repository entity --- bothub/common/tests.py | 58 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/bothub/common/tests.py b/bothub/common/tests.py index df29efed8..5c709230c 100644 --- a/bothub/common/tests.py +++ b/bothub/common/tests.py @@ -11,12 +11,13 @@ from .models import RepositoryTranslatedExample from .models import RepositoryTranslatedExampleEntity from .models import RepositoryAuthorization -from .models import DoesNotHaveTranslation from .models import RequestRepositoryAuthorization +from .models import RepositoryEntity from . import languages from .exceptions import RepositoryUpdateAlreadyStartedTraining from .exceptions import RepositoryUpdateAlreadyTrained from .exceptions import TrainingNotAllowed +from .exceptions import DoesNotHaveTranslation class RepositoryUpdateTestCase(TestCase): @@ -794,3 +795,58 @@ def test_approve_twice_another_admin(self): with self.assertRaises(ValidationError): self.ra.approved_by = self.admin self.ra.save() + + +class RepositoryEntityTestCase(TestCase): + def setUp(self): + self.language = languages.LANGUAGE_EN + + self.owner = User.objects.create_user('owner@user.com', 'user') + + self.repository = Repository.objects.create( + owner=self.owner, + name='Test', + slug='test', + language=self.language) + + self.example = RepositoryExample.objects.create( + repository_update=self.repository.current_update(), + text='my name is Douglas') + + self.example_entity_1 = RepositoryExampleEntity.objects.create( + repository_example=self.example, + start=11, + end=18, + entity='name') + + self.example_entity_2 = RepositoryExampleEntity.objects.create( + repository_example=self.example, + start=0, + end=2, + entity='object') + + def test_example_entity_create_entity(self): + name_entity = RepositoryEntity.objects.get( + repository=self.repository, + value='name') + self.assertEqual( + name_entity.pk, + self.example_entity_1.pk) + + def test_example_entity_dont_duplicate_entity(self): + name_entity = RepositoryEntity.objects.get( + repository=self.repository, + value='name') + + new_example_entity = RepositoryExampleEntity.objects.create( + repository_example=self.example, + start=11, + end=18, + entity='name') + + self.assertEqual( + name_entity.pk, + self.example_entity_1.pk) + self.assertEqual( + name_entity.pk, + new_example_entity.entity.pk) From 865ac7ddc24d43acf30a357091c811848343a831 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Wed, 25 Jul 2018 15:16:49 -0300 Subject: [PATCH 05/19] Add label field to RepositoryEntity --- .../migrations/0019_auto_20180725_1657.py | 41 +++++++++ bothub/common/models.py | 64 ++++++++++++-- bothub/common/tests.py | 87 ++++++++++++++++++- 3 files changed, 186 insertions(+), 6 deletions(-) create mode 100644 bothub/common/migrations/0019_auto_20180725_1657.py diff --git a/bothub/common/migrations/0019_auto_20180725_1657.py b/bothub/common/migrations/0019_auto_20180725_1657.py new file mode 100644 index 000000000..f5a64ccc3 --- /dev/null +++ b/bothub/common/migrations/0019_auto_20180725_1657.py @@ -0,0 +1,41 @@ +# Generated by Django 2.0.6 on 2018-07-25 16:57 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import re + + +class Migration(migrations.Migration): + + dependencies = [ + ('common', '0018_auto_20180725_1305'), + ] + + operations = [ + migrations.CreateModel( + name='RepositoryEntityLabel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('value', models.CharField(blank=True, max_length=64, validators=[django.core.validators.RegexValidator(re.compile('^[-a-z0-9_]+\\Z'), 'Enter a valid value consisting of lowercase letters, numbers, underscores or hyphens.', 'invalid')], verbose_name='label')), + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='created at')), + ('repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='labels', to='common.Repository')), + ], + ), + migrations.AddField( + model_name='repositoryentity', + name='created_at', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now, verbose_name='created at'), + preserve_default=False, + ), + migrations.AddField( + model_name='repositoryentity', + name='label', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='entities', to='common.RepositoryEntityLabel'), + ), + migrations.AlterUniqueTogether( + name='repositoryentitylabel', + unique_together={('repository', 'value')}, + ), + ] diff --git a/bothub/common/models.py b/bothub/common/models.py index 3091398dd..f6a2829a3 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -23,9 +23,9 @@ from .exceptions import DoesNotHaveTranslation -entity_and_intent_regex = _lazy_re_compile(r'^[-a-z0-9_]+\Z') -validate_entity_and_intent = RegexValidator( - entity_and_intent_regex, +item_key_regex = _lazy_re_compile(r'^[-a-z0-9_]+\Z') +validate_item_key = RegexValidator( + item_key_regex, _('Enter a valid value consisting of lowercase letters, numbers, ' + 'underscores or hyphens.'), 'invalid' @@ -409,7 +409,7 @@ class Meta: max_length=64, blank=True, help_text=_('Example intent reference'), - validators=[validate_entity_and_intent]) + validators=[validate_item_key]) created_at = models.DateTimeField( _('created at'), auto_now_add=True) @@ -542,6 +542,43 @@ def has_valid_entities(self): list(map(lambda x: x.to_dict, my_entities))) +class RepositoryEntityLabelQueryset(models.QuerySet): + def get(self, repository, value): + try: + return super().get( + repository=repository, + value=value) + except self.model.DoesNotExist as e: + return super().create( + repository=repository, + value=value) + + +class RepositoryEntityLabelManager(models.Manager): + def get_queryset(self): + return RepositoryEntityLabelQueryset(self.model, using=self._db) + + +class RepositoryEntityLabel(models.Model): + class Meta: + unique_together = ['repository', 'value'] + + repository = models.ForeignKey( + Repository, + on_delete=models.CASCADE, + related_name='labels') + value = models.CharField( + _('label'), + max_length=64, + validators=[validate_item_key], + blank=True) + created_at = models.DateTimeField( + _('created at'), + auto_now_add=True) + + objects = RepositoryEntityLabelManager() + + class RepositoryEntityQueryset(models.QuerySet): def get(self, repository, value): try: @@ -571,10 +608,27 @@ class Meta: _('entity'), max_length=64, help_text=_('Entity name'), - validators=[validate_entity_and_intent]) + validators=[validate_item_key]) + label = models.ForeignKey( + RepositoryEntityLabel, + on_delete=models.CASCADE, + related_name='entities', + null=True, + blank=True) + created_at = models.DateTimeField( + _('created at'), + auto_now_add=True) objects = RepositoryEntityManager() + def set_label(self, value): + if not value: + self.label = None + else: + self.label = RepositoryEntityLabel.objects.get( + repository=self.repository, + value=value) + class EntityBaseQueryset(models.QuerySet): def create(self, entity, **kwargs): diff --git a/bothub/common/tests.py b/bothub/common/tests.py index 5c709230c..5df487041 100644 --- a/bothub/common/tests.py +++ b/bothub/common/tests.py @@ -13,6 +13,7 @@ from .models import RepositoryAuthorization from .models import RequestRepositoryAuthorization from .models import RepositoryEntity +from .models import RepositoryEntityLabel from . import languages from .exceptions import RepositoryUpdateAlreadyStartedTraining from .exceptions import RepositoryUpdateAlreadyTrained @@ -833,7 +834,7 @@ def test_example_entity_create_entity(self): name_entity.pk, self.example_entity_1.pk) - def test_example_entity_dont_duplicate_entity(self): + def test_dont_duplicate_entity(self): name_entity = RepositoryEntity.objects.get( repository=self.repository, value='name') @@ -850,3 +851,87 @@ def test_example_entity_dont_duplicate_entity(self): self.assertEqual( name_entity.pk, new_example_entity.entity.pk) + + +class RepositoryEntityLabelTestCase(TestCase): + def setUp(self): + self.language = languages.LANGUAGE_EN + + self.owner = User.objects.create_user('owner@user.com', 'user') + + self.repository = Repository.objects.create( + owner=self.owner, + name='Test', + slug='test', + language=self.language) + + self.example = RepositoryExample.objects.create( + repository_update=self.repository.current_update(), + text='my name is Douglas') + + self.example_entity_1 = RepositoryExampleEntity.objects.create( + repository_example=self.example, + start=11, + end=18, + entity='name') + + self.example_entity_2 = RepositoryExampleEntity.objects.create( + repository_example=self.example, + start=0, + end=2, + entity='object') + + def test_set_label(self): + name_entity = RepositoryEntity.objects.get( + repository=self.repository, + value='name') + + name_entity.set_label('subject') + + self.assertIsNotNone(name_entity.label) + + def test_entity_label_created(self): + name_entity = RepositoryEntity.objects.get( + repository=self.repository, + value='name') + + name_entity.set_label('subject') + + subject_label = RepositoryEntityLabel.objects.get( + repository=self.repository, + value='subject') + + self.assertEqual( + name_entity.label.pk, + subject_label.pk) + + def test_dont_duplicate_label(self): + name_entity = RepositoryEntity.objects.get( + repository=self.repository, + value='name') + name_entity.set_label('subject') + + object_entity = RepositoryEntity.objects.get( + repository=self.repository, + value='object') + object_entity.set_label('subject') + + subject_label = RepositoryEntityLabel.objects.get( + repository=self.repository, + value='subject') + + self.assertEqual( + name_entity.label.pk, + subject_label.pk) + self.assertEqual( + object_entity.label.pk, + subject_label.pk) + + def test_set_label_to_none(self): + name_entity = RepositoryEntity.objects.get( + repository=self.repository, + value='name') + + name_entity.set_label(None) + + self.assertIsNone(name_entity.label) From 384466750c7a3b8323732d516ab4387f394aa786 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Wed, 25 Jul 2018 16:05:33 -0300 Subject: [PATCH 06/19] Add label in RepositoryExampleEntitySerializer --- bothub/api/serializers/example.py | 9 ++++++--- bothub/api/tests/test_example.py | 32 ++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/bothub/api/serializers/example.py b/bothub/api/serializers/example.py index f97542554..43a5e4b0b 100644 --- a/bothub/api/serializers/example.py +++ b/bothub/api/serializers/example.py @@ -23,6 +23,7 @@ class Meta: 'start', 'end', 'entity', + 'label', 'created_at', 'value', ] @@ -34,13 +35,15 @@ class Meta: ], help_text=_('Example\'s ID')) entity = serializers.SerializerMethodField() - value = serializers.SerializerMethodField() + label = serializers.SerializerMethodField() def get_entity(self, obj): return obj.entity.value - def get_value(self, obj): - return obj.value + def get_label(self, obj): + if not obj.entity.label: + return None + return obj.entity.label.value class NewRepositoryExampleEntitySerializer(serializers.ModelSerializer): diff --git a/bothub/api/tests/test_example.py b/bothub/api/tests/test_example.py index 91f1ecf07..3db024bc4 100644 --- a/bothub/api/tests/test_example.py +++ b/bothub/api/tests/test_example.py @@ -208,7 +208,7 @@ def setUp(self): self.example = RepositoryExample.objects.create( repository_update=self.repository.current_update(), text='my name is douglas') - RepositoryExampleEntity.objects.create( + self.example_entity = RepositoryExampleEntity.objects.create( repository_example=self.example, start=11, end=18, @@ -278,6 +278,36 @@ def test_list_entities(self): len(content_data.get('entities')), 1) + def test_entity_has_label(self): + response, content_data = self.request( + self.example, + self.owner_token) + self.assertEqual( + response.status_code, + status.HTTP_200_OK) + entity = content_data.get('entities')[0] + self.assertIn( + 'label', + entity.keys()) + + def test_entity_has_valid_label(self): + label = 'subject' + self.example_entity.entity.set_label('subject') + self.example_entity.entity.save(update_fields=['label']) + response, content_data = self.request( + self.example, + self.owner_token) + self.assertEqual( + response.status_code, + status.HTTP_200_OK) + entity = content_data.get('entities')[0] + self.assertIn( + 'label', + entity.keys()) + self.assertEqual( + entity.get('label'), + label) + class RepositoryExampleDestroyTestCase(TestCase): def setUp(self): From 776948c8a9fae8032c4bb61162e246fbd505922e Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Thu, 26 Jul 2018 09:35:25 -0300 Subject: [PATCH 07/19] Add entity label in NewRepositoryExampleEntitySerializer --- bothub/api/fields.py | 18 ++++++++++++++++++ bothub/api/serializers/example.py | 28 +++++++++++++++++++++++++--- bothub/api/tests/test_example.py | 23 +++++++++++++++++++++++ 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/bothub/api/fields.py b/bothub/api/fields.py index 79e7f616f..b41552d3d 100644 --- a/bothub/api/fields.py +++ b/bothub/api/fields.py @@ -1,6 +1,7 @@ from rest_framework import serializers from bothub.common.models import RepositoryEntity +from bothub.common.models import RepositoryEntityLabel class ModelMultipleChoiceField(serializers.ManyRelatedField): @@ -36,3 +37,20 @@ def __init__(self, *args, validators=[], **kwargs): def to_representation(self, obj): return obj.value + + +class LabelValueField(serializers.CharField): + def __init__(self, *args, validators=[], **kwargs): + kwargs.pop('max_length', 0) + kwargs.pop('help_text', '') + + value_field = RepositoryEntityLabel._meta.get_field('value') + + super().__init__( + *args, + max_length=value_field.max_length, + validators=(validators + value_field.validators), + **kwargs) + + def to_representation(self, obj): + return obj.value diff --git a/bothub/api/serializers/example.py b/bothub/api/serializers/example.py index 43a5e4b0b..d9d7e4548 100644 --- a/bothub/api/serializers/example.py +++ b/bothub/api/serializers/example.py @@ -1,4 +1,5 @@ from rest_framework import serializers +from rest_framework.fields import empty from django.utils.translation import gettext as _ @@ -8,6 +9,7 @@ from ..fields import EntityText from ..fields import EntityValueField +from ..fields import LabelValueField from ..validators import CanContributeInRepositoryExampleValidator from ..validators import CanContributeInRepositoryValidator from ..validators import ExampleWithIntentOrEntityValidator @@ -54,9 +56,28 @@ class Meta: 'start', 'end', 'entity', + 'entity_label', ] + repository_example = serializers.PrimaryKeyRelatedField( + queryset=RepositoryExample.objects, + required=False) + entity = EntityValueField() + entity_label = LabelValueField( + allow_blank=True, + required=False) + + def create(self, validated_data): + repository_example = validated_data.pop('repository_example', None) + assert repository_example + entity_label = validated_data.pop('entity_label', empty) + example_entity = self.Meta.model.objects.create( + repository_example=repository_example, + **validated_data) + if entity_label is not empty: + example_entity.entity.set_label(entity_label) + return example_entity class RepositoryExampleSerializer(serializers.ModelSerializer): @@ -131,7 +152,8 @@ def create(self, validated_data): entities_data = validated_data.pop('entities') example = self.Meta.model.objects.create(**validated_data) for entity_data in entities_data: - RepositoryExampleEntity.objects.create( - repository_example=example, - **entity_data) + entity_data.update({'repository_example': example.pk}) + entity_serializer = NewRepositoryExampleEntitySerializer(data=entity_data) + entity_serializer.is_valid(raise_exception=True) + entity_serializer.save() return example diff --git a/bothub/api/tests/test_example.py b/bothub/api/tests/test_example.py index 3db024bc4..a7c200ac8 100644 --- a/bothub/api/tests/test_example.py +++ b/bothub/api/tests/test_example.py @@ -141,6 +141,29 @@ def test_with_entities(self): len(content_data.get('entities')), 1) + def test_with_entities_with_label(self): + response, content_data = self.request( + self.owner_token, + { + 'repository': str(self.repository.uuid), + 'text': 'my name is douglas', + 'intent': 'greet', + 'entities': [ + { + 'start': 11, + 'end': 18, + 'entity': 'name', + 'label': 'subject', + }, + ], + }) + self.assertEqual( + response.status_code, + status.HTTP_201_CREATED) + self.assertEqual( + len(content_data.get('entities')), + 1) + def test_intent_or_entity_required(self): response, content_data = self.request( self.owner_token, From d1094e8e520c6ecf68564fc0103c1d9f823fc237 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Thu, 26 Jul 2018 10:02:28 -0300 Subject: [PATCH 08/19] pep8 --- Makefile | 2 +- bothub/api/serializers/example.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 769f305ff..5f853b6f7 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ install_requirements: lint: @make development_mode_guard @make check_environment - @pipenv run flake8 + @PIPENV_DONT_LOAD_ENV=1 pipenv run flake8 @echo "${SUCCESS}✔${NC} The code is following the PEP8" test: diff --git a/bothub/api/serializers/example.py b/bothub/api/serializers/example.py index d9d7e4548..1c6602efd 100644 --- a/bothub/api/serializers/example.py +++ b/bothub/api/serializers/example.py @@ -153,7 +153,8 @@ def create(self, validated_data): example = self.Meta.model.objects.create(**validated_data) for entity_data in entities_data: entity_data.update({'repository_example': example.pk}) - entity_serializer = NewRepositoryExampleEntitySerializer(data=entity_data) + entity_serializer = NewRepositoryExampleEntitySerializer( + data=entity_data) entity_serializer.is_valid(raise_exception=True) entity_serializer.save() return example From 5a387b16e936d0d3684cb18975423d237616bfb8 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Wed, 1 Aug 2018 14:21:00 -0300 Subject: [PATCH 09/19] Add RepositoryEntitySerializer --- bothub/api/serializers/__init__.py | 1 + bothub/api/serializers/example.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/bothub/api/serializers/__init__.py b/bothub/api/serializers/__init__.py index 0623947d6..a28c6147f 100644 --- a/bothub/api/serializers/__init__.py +++ b/bothub/api/serializers/__init__.py @@ -17,6 +17,7 @@ RepositoryExampleSerializer, NewRepositoryExampleSerializer, NewRepositoryExampleEntitySerializer, + RepositoryEntitySerializer, ) from .translate import ( # noqa: F401 RepositoryTranslatedExampleEntitySeralizer, diff --git a/bothub/api/serializers/example.py b/bothub/api/serializers/example.py index 1c6602efd..405c1062f 100644 --- a/bothub/api/serializers/example.py +++ b/bothub/api/serializers/example.py @@ -6,6 +6,7 @@ from bothub.common.models import Repository from bothub.common.models import RepositoryExample from bothub.common.models import RepositoryExampleEntity +from bothub.common.models import RepositoryEntity from ..fields import EntityText from ..fields import EntityValueField @@ -158,3 +159,20 @@ def create(self, validated_data): entity_serializer.is_valid(raise_exception=True) entity_serializer.save() return example + + +class RepositoryEntitySerializer(serializers.ModelSerializer): + class Meta: + model = RepositoryEntity + fields = [ + 'repository', + 'value', + 'label', + ] + + label = serializers.SerializerMethodField() + + def get_label(self, obj): + if not obj.label: + return None + return obj.label.value From f4b336f4be516cb10ed2d31b31b2782901b876b2 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Wed, 1 Aug 2018 14:21:29 -0300 Subject: [PATCH 10/19] Add entities endpoint in api, add RepositoryEntitiesTestCase --- bothub/api/routers.py | 2 + bothub/api/tests/test_example.py | 74 ++++++++++++++++++++++++++++++++ bothub/api/views.py | 55 ++++++++++++++++++++++++ 3 files changed, 131 insertions(+) diff --git a/bothub/api/routers.py b/bothub/api/routers.py index f6387e9d0..f038c2afc 100644 --- a/bothub/api/routers.py +++ b/bothub/api/routers.py @@ -24,6 +24,7 @@ from .views import RequestAuthorizationViewSet from .views import RepositoryAuthorizationRequestsViewSet from .views import ReviewAuthorizationRequestViewSet +from .views import RepositoryEntitiesViewSet class Router(routers.SimpleRouter): @@ -119,3 +120,4 @@ def get_lookup_regex(self, viewset, lookup_prefix=''): RepositoryAuthorizationRequestsViewSet) router.register('review-authorization-request', ReviewAuthorizationRequestViewSet) +router.register('entities', RepositoryEntitiesViewSet) diff --git a/bothub/api/tests/test_example.py b/bothub/api/tests/test_example.py index a7c200ac8..62a9c2c0c 100644 --- a/bothub/api/tests/test_example.py +++ b/bothub/api/tests/test_example.py @@ -12,6 +12,7 @@ from ..views import NewRepositoryExampleViewSet from ..views import RepositoryExampleViewSet +from ..views import RepositoryEntitiesViewSet from .utils import create_user_and_token @@ -410,3 +411,76 @@ def test_already_deleted(self): self.assertEqual( response.status_code, status.HTTP_500_INTERNAL_SERVER_ERROR) + + +class RepositoryEntitiesTestCase(TestCase): + def setUp(self): + self.factory = RequestFactory() + + self.owner, self.owner_token = create_user_and_token('owner') + self.user, self.user_token = create_user_and_token() + + self.entity_value = 'douglas' + + self.repository = Repository.objects.create( + owner=self.owner, + name='Testing', + slug='test', + language=languages.LANGUAGE_EN) + self.example = RepositoryExample.objects.create( + repository_update=self.repository.current_update(), + text='my name is douglas') + self.example_entity = RepositoryExampleEntity.objects.create( + repository_example=self.example, + start=11, + end=18, + entity=self.entity_value) + self.example_entity.entity.set_label('name') + self.example_entity.entity.save() + + def request(self, data, token): + authorization_header = { + 'HTTP_AUTHORIZATION': 'Token {}'.format(token.key), + } + request = self.factory.get( + '/api/entities/', + data=data, + **authorization_header) + response = RepositoryEntitiesViewSet.as_view( + {'get': 'list'})(request) + response.render() + content_data = json.loads(response.content) + return (response, content_data,) + + def test_okay(self): + response, content_data = self.request( + { + 'repository_uuid': self.repository.uuid, + }, + self.owner_token) + self.assertEqual( + response.status_code, + status.HTTP_200_OK) + self.assertEqual(content_data.get('count'), 1) + + response, content_data = self.request( + { + 'repository_uuid': self.repository.uuid, + 'value': self.entity_value, + }, + self.owner_token) + self.assertEqual( + response.status_code, + status.HTTP_200_OK) + self.assertEqual(content_data.get('count'), 1) + + response, content_data = self.request( + { + 'repository_uuid': self.repository.uuid, + 'value': 'other', + }, + self.owner_token) + self.assertEqual( + response.status_code, + status.HTTP_200_OK) + self.assertEqual(content_data.get('count'), 0) diff --git a/bothub/api/views.py b/bothub/api/views.py index 7b6b89317..fd437c12a 100644 --- a/bothub/api/views.py +++ b/bothub/api/views.py @@ -27,6 +27,7 @@ from bothub.common.models import RepositoryVote from bothub.common.models import RepositoryAuthorization from bothub.common.models import RequestRepositoryAuthorization +from bothub.common.models import RepositoryEntity from bothub.authentication.models import User from .serializers import RepositorySerializer @@ -50,6 +51,7 @@ from .serializers import NewRequestRepositoryAuthorizationSerializer from .serializers import RequestRepositoryAuthorizationSerializer from .serializers import ReviewAuthorizationRequestSerializer +from .serializers import RepositoryEntitySerializer # Permisions @@ -105,6 +107,18 @@ def has_object_permission(self, request, view, obj): return authorization.is_admin +class RepositoryEntityHasPermission(permissions.BasePermission): + def has_object_permission(self, request, view, obj): + authorization = obj.repository.get_user_authorization(request.user) + if request.method in READ_METHODS: + return authorization.can_read + if request.user.is_authenticated: + if request.method in WRITE_METHODS: + return authorization.can_write + return authorization.is_admin + return False + + # Filters class ExamplesFilter(filters.FilterSet): @@ -284,6 +298,35 @@ def filter_repository_uuid(self, queryset, name, value): raise NotFound(_('Invalid repository UUID')) +class RepositoryEntitiesFilter(filters.FilterSet): + class Meta: + model = RepositoryEntity + fields = [ + 'repository_uuid', + 'value', + ] + + repository_uuid = filters.CharFilter( + name='repository_uuid', + required=True, + method='filter_repository_uuid', + help_text=_('Repository\'s UUID')) + + def filter_repository_uuid(self, queryset, name, value): + request = self.request + try: + repository = Repository.objects.get(uuid=value) + authorization = repository.get_user_authorization(request.user) + if not authorization.is_admin: + raise PermissionDenied() + return queryset.filter(repository=repository) + except Repository.DoesNotExist: + raise NotFound( + _('Repository {} does not exist').format(value)) + except DjangoValidationError: + raise NotFound(_('Invalid repository UUID')) + + # Mixins class MultipleFieldLookupMixin(object): @@ -896,3 +939,15 @@ def update(self, *args, **kwargs): return super().update(*args, **kwargs) except DjangoValidationError as e: raise ValidationError(e.message) + + +class RepositoryEntitiesViewSet( + mixins.ListModelMixin, + GenericViewSet): + queryset = RepositoryEntity.objects.all() + serializer_class = RepositoryEntitySerializer + filter_class = RepositoryEntitiesFilter + permission_classes = [ + IsAuthenticated, + RepositoryEntityHasPermission, + ] From 1e802e9e074364b63a9b9b8a2a9a171fc077c971 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Fri, 3 Aug 2018 08:25:56 -0300 Subject: [PATCH 11/19] Update Makefile to linux support --- Makefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 5f853b6f7..83083d20d 100644 --- a/Makefile +++ b/Makefile @@ -10,10 +10,10 @@ help: @exit 0 check_environment: - @if [[ ${CHECK_ENVIRONMENT} = true ]]; then make _check_environment; fi + @if [ ${CHECK_ENVIRONMENT} = true ]; then make _check_environment; fi install_requirements: - @if [[ ${IS_PRODUCTION} = true ]]; \ + @if [ ${IS_PRODUCTION} = true ]; \ then make install_production_requirements; \ else make install_development_requirements; fi @@ -33,7 +33,7 @@ test: migrate: @make check_environment - @if [[ ${IS_PRODUCTION} = true ]]; \ + @if [ ${IS_PRODUCTION} = true ]; \ then python manage.py migrate; \ else pipenv run python manage.py migrate; fi @@ -52,7 +52,7 @@ migrations: collectstatic: @make check_environment - @if [[ ${IS_PRODUCTION} = true ]]; \ + @if [ ${IS_PRODUCTION} = true ]; \ then python manage.py collectstatic --no-input; \ else pipenv run python manage.py collectstatic --no-input; fi @@ -83,8 +83,8 @@ install_production_requirements: @echo "${SUCCESS}✔${NC} Requirements installed" development_mode_guard: - @if [[ ${IS_PRODUCTION} = true ]]; then echo "${DANGER}Just run this command in development mode${NC}"; fi - @if [[ ${IS_PRODUCTION} = true ]]; then exit 1; fi + @if [ ${IS_PRODUCTION} = true ]; then echo "${DANGER}Just run this command in development mode${NC}"; fi + @if [ ${IS_PRODUCTION} = true ]; then exit 1; fi # Checkers @@ -92,6 +92,6 @@ development_mode_guard: _check_environment: @type pipenv &> /dev/null || (echo "${DANGER}☓${NC} Install pipenv to continue..." && exit 1) @echo "${SUCCESS}✔${NC} pipenv installed" - @if [[ ! -f "${ENVIRONMENT_VARS_FILE}" && ${IS_PRODUCTION} = false ]]; then make create_environment_vars_file; fi + @if [ ! -f "${ENVIRONMENT_VARS_FILE}" && ${IS_PRODUCTION} = false ]; then make create_environment_vars_file; fi @make install_requirements @echo "${SUCCESS}✔${NC} Environment checked" \ No newline at end of file From a554c2471516e6806df57399099a7428a6f043fe Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Mon, 6 Aug 2018 10:11:12 -0300 Subject: [PATCH 12/19] Undo Makefiles changes --- Makefile | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 83083d20d..115c896b6 100644 --- a/Makefile +++ b/Makefile @@ -10,10 +10,10 @@ help: @exit 0 check_environment: - @if [ ${CHECK_ENVIRONMENT} = true ]; then make _check_environment; fi + @if [[ ${CHECK_ENVIRONMENT} = true ]]; then make _check_environment; fi install_requirements: - @if [ ${IS_PRODUCTION} = true ]; \ + @if [[ ${IS_PRODUCTION} = true ]]; \ then make install_production_requirements; \ else make install_development_requirements; fi @@ -33,7 +33,7 @@ test: migrate: @make check_environment - @if [ ${IS_PRODUCTION} = true ]; \ + @if [[ ${IS_PRODUCTION} = true ]]; \ then python manage.py migrate; \ else pipenv run python manage.py migrate; fi @@ -52,7 +52,7 @@ migrations: collectstatic: @make check_environment - @if [ ${IS_PRODUCTION} = true ]; \ + @if [[ ${IS_PRODUCTION} = true ]]; \ then python manage.py collectstatic --no-input; \ else pipenv run python manage.py collectstatic --no-input; fi @@ -83,8 +83,8 @@ install_production_requirements: @echo "${SUCCESS}✔${NC} Requirements installed" development_mode_guard: - @if [ ${IS_PRODUCTION} = true ]; then echo "${DANGER}Just run this command in development mode${NC}"; fi - @if [ ${IS_PRODUCTION} = true ]; then exit 1; fi + @if [[ ${IS_PRODUCTION} = true ]]; then echo "${DANGER}Just run this command in development mode${NC}"; fi + @if [[ ${IS_PRODUCTION} = true ]]; then exit 1; fi # Checkers @@ -92,6 +92,7 @@ development_mode_guard: _check_environment: @type pipenv &> /dev/null || (echo "${DANGER}☓${NC} Install pipenv to continue..." && exit 1) @echo "${SUCCESS}✔${NC} pipenv installed" - @if [ ! -f "${ENVIRONMENT_VARS_FILE}" && ${IS_PRODUCTION} = false ]; then make create_environment_vars_file; fi + @if [[ ! -f "${ENVIRONMENT_VARS_FILE}" && ${IS_PRODUCTION} = false ]]; \ + then make create_environment_vars_file; fi @make install_requirements @echo "${SUCCESS}✔${NC} Environment checked" \ No newline at end of file From ce25a1dd31760f45b20e6e3fb5df6d96defe06a1 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Tue, 7 Aug 2018 12:29:45 -0300 Subject: [PATCH 13/19] Fix new example with label --- bothub/api/serializers/example.py | 11 ++++++----- bothub/api/tests/test_example.py | 4 ++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/bothub/api/serializers/example.py b/bothub/api/serializers/example.py index 405c1062f..72bed0026 100644 --- a/bothub/api/serializers/example.py +++ b/bothub/api/serializers/example.py @@ -57,7 +57,7 @@ class Meta: 'start', 'end', 'entity', - 'entity_label', + 'label', ] repository_example = serializers.PrimaryKeyRelatedField( @@ -65,19 +65,20 @@ class Meta: required=False) entity = EntityValueField() - entity_label = LabelValueField( + label = LabelValueField( allow_blank=True, required=False) def create(self, validated_data): repository_example = validated_data.pop('repository_example', None) assert repository_example - entity_label = validated_data.pop('entity_label', empty) + label = validated_data.pop('label', empty) example_entity = self.Meta.model.objects.create( repository_example=repository_example, **validated_data) - if entity_label is not empty: - example_entity.entity.set_label(entity_label) + if label is not empty: + example_entity.entity.set_label(label) + example_entity.entity.save(update_fields=['label']) return example_entity diff --git a/bothub/api/tests/test_example.py b/bothub/api/tests/test_example.py index 62a9c2c0c..7c1739112 100644 --- a/bothub/api/tests/test_example.py +++ b/bothub/api/tests/test_example.py @@ -164,6 +164,10 @@ def test_with_entities_with_label(self): self.assertEqual( len(content_data.get('entities')), 1) + id = content_data.get('id') + repository_example = RepositoryExample.objects.get(id=id) + example_entity = repository_example.entities.all()[0] + self.assertIsNotNone(example_entity.entity.label) def test_intent_or_entity_required(self): response, content_data = self.request( From 91768d0f42fc352be0d754a9cf9316a3cdf4a475 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Fri, 10 Aug 2018 09:31:09 -0300 Subject: [PATCH 14/19] bothub engine --- bothub/common/models.py | 22 +++++++++------------- bothub/common/tests.py | 14 -------------- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/bothub/common/models.py b/bothub/common/models.py index f6a2829a3..d6a9088ad 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -439,15 +439,6 @@ def get_entities(self, language): return self.entities.all() return self.get_translation(language).entities.all() - def rasa_nlu_data(self, language): - return { - 'text': self.get_text(language), - 'intent': self.intent, - 'entities': [ - entity.rasa_nlu_data for entity in self.get_entities( - language)], - } - def delete(self): self.deleted_in = self.repository_update.repository.current_update( self.repository_update.language) @@ -688,15 +679,20 @@ def rasa_nlu_data(self): @property def to_dict(self): + return self.get_rasa_nlu_data() + + def get_example(self): + pass # pragma: no cover + + + def get_rasa_nlu_data(self, label_as_entity=False): return { 'start': self.start, 'end': self.end, - 'entity': self.entity.value, + 'entity': self.entity.label.value + if label_as_entity else self.entity.value, } - def get_example(self): - pass # pragma: no cover - class RepositoryExampleEntity(EntityBase): repository_example = models.ForeignKey( diff --git a/bothub/common/tests.py b/bothub/common/tests.py index 5df487041..96105cdf8 100644 --- a/bothub/common/tests.py +++ b/bothub/common/tests.py @@ -90,17 +90,6 @@ def test_new_translate(self): len(self.repository.current_update(language).examples), 1) - def test_to_rasa_nlu_data(self): - language = languages.LANGUAGE_PT - RepositoryTranslatedExample.objects.create( - original_example=self.example, - language=language, - text='meu nome é Douglas') - - self.assertDictEqual( - self.example.rasa_nlu_data(language), - TranslateTestCase.EXPECTED_RASA_NLU_DATA) - def test_translated_entity(self): RepositoryExampleEntity.objects.create( repository_example=self.example, @@ -118,9 +107,6 @@ def test_translated_entity(self): start=11, end=18, entity='name') - self.assertDictEqual( - self.example.rasa_nlu_data(language), - TranslateTestCase.EXPECTED_RASA_NLU_DATA_WITH_ENTITIES) def test_valid_entities(self): RepositoryExampleEntity.objects.create( From b899e1375b772b8576d1fd33240b0cab4bf1a615 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Fri, 10 Aug 2018 13:30:49 -0300 Subject: [PATCH 15/19] pep8 --- bothub/common/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bothub/common/models.py b/bothub/common/models.py index d6a9088ad..cb11b8564 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -683,7 +683,6 @@ def to_dict(self): def get_example(self): pass # pragma: no cover - def get_rasa_nlu_data(self, label_as_entity=False): return { From caa3aad62e7b6c9d423291f21dd4b85a3be2e300 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Mon, 13 Aug 2018 10:32:02 -0300 Subject: [PATCH 16/19] Add new label field validator: can_t_be_other --- bothub/api/tests/test_example.py | 27 +++++++++++++++++++ .../migrations/0020_auto_20180813_1320.py | 21 +++++++++++++++ bothub/common/models.py | 11 ++++++-- 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 bothub/common/migrations/0020_auto_20180813_1320.py diff --git a/bothub/api/tests/test_example.py b/bothub/api/tests/test_example.py index 7c1739112..4c2910ba5 100644 --- a/bothub/api/tests/test_example.py +++ b/bothub/api/tests/test_example.py @@ -169,6 +169,33 @@ def test_with_entities_with_label(self): example_entity = repository_example.entities.all()[0] self.assertIsNotNone(example_entity.entity.label) + def test_with_entities_with_invalid_label(self): + response, content_data = self.request( + self.owner_token, + { + 'repository': str(self.repository.uuid), + 'text': 'my name is douglas', + 'intent': 'greet', + 'entities': [ + { + 'start': 11, + 'end': 18, + 'entity': 'name', + 'label': 'other', + }, + ], + }) + self.assertEqual( + response.status_code, + status.HTTP_400_BAD_REQUEST) + self.assertIn( + 'entities', + content_data.keys()) + entities_errors = content_data.get('entities') + self.assertIn( + 'label', + entities_errors[0]) + def test_intent_or_entity_required(self): response, content_data = self.request( self.owner_token, diff --git a/bothub/common/migrations/0020_auto_20180813_1320.py b/bothub/common/migrations/0020_auto_20180813_1320.py new file mode 100644 index 000000000..ebe2eb388 --- /dev/null +++ b/bothub/common/migrations/0020_auto_20180813_1320.py @@ -0,0 +1,21 @@ +# Generated by Django 2.0.6 on 2018-08-13 13:20 + +import bothub.common.models +import django.core.validators +from django.db import migrations, models +import re + + +class Migration(migrations.Migration): + + dependencies = [ + ('common', '0019_auto_20180725_1657'), + ] + + operations = [ + migrations.AlterField( + model_name='repositoryentitylabel', + name='value', + field=models.CharField(blank=True, max_length=64, validators=[django.core.validators.RegexValidator(re.compile('^[-a-z0-9_]+\\Z'), 'Enter a valid value consisting of lowercase letters, numbers, underscores or hyphens.', 'invalid'), bothub.common.models.can_t_be_other], verbose_name='label'), + ), + ] diff --git a/bothub/common/models.py b/bothub/common/models.py index d6a9088ad..a93d44147 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -32,6 +32,11 @@ ) +def can_t_be_other(value): + if value == 'other': + raise ValidationError(_('The label can\'t be named as "other"')) + + class RepositoryCategory(models.Model): class Meta: verbose_name = _('repository category') @@ -561,7 +566,10 @@ class Meta: value = models.CharField( _('label'), max_length=64, - validators=[validate_item_key], + validators=[ + validate_item_key, + can_t_be_other, + ], blank=True) created_at = models.DateTimeField( _('created at'), @@ -683,7 +691,6 @@ def to_dict(self): def get_example(self): pass # pragma: no cover - def get_rasa_nlu_data(self, label_as_entity=False): return { From 5de99e6226161b10a664e2130ae387a2061bc3b0 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Mon, 13 Aug 2018 10:48:33 -0300 Subject: [PATCH 17/19] Add entity not equal label validator --- bothub/api/serializers/example.py | 5 +++++ bothub/api/tests/test_example.py | 27 +++++++++++++++++++++++++++ bothub/api/validators.py | 10 ++++++++++ 3 files changed, 42 insertions(+) diff --git a/bothub/api/serializers/example.py b/bothub/api/serializers/example.py index 72bed0026..cff4bd28e 100644 --- a/bothub/api/serializers/example.py +++ b/bothub/api/serializers/example.py @@ -14,6 +14,7 @@ from ..validators import CanContributeInRepositoryExampleValidator from ..validators import CanContributeInRepositoryValidator from ..validators import ExampleWithIntentOrEntityValidator +from ..validators import EntityNotEqualLabelValidator from .translate import RepositoryTranslatedExampleSerializer @@ -69,6 +70,10 @@ class Meta: allow_blank=True, required=False) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.validators.append(EntityNotEqualLabelValidator()) + def create(self, validated_data): repository_example = validated_data.pop('repository_example', None) assert repository_example diff --git a/bothub/api/tests/test_example.py b/bothub/api/tests/test_example.py index 4c2910ba5..b034d4842 100644 --- a/bothub/api/tests/test_example.py +++ b/bothub/api/tests/test_example.py @@ -196,6 +196,33 @@ def test_with_entities_with_invalid_label(self): 'label', entities_errors[0]) + def test_with_entities_with_equal_label(self): + response, content_data = self.request( + self.owner_token, + { + 'repository': str(self.repository.uuid), + 'text': 'my name is douglas', + 'intent': 'greet', + 'entities': [ + { + 'start': 11, + 'end': 18, + 'entity': 'name', + 'label': 'name', + }, + ], + }) + self.assertEqual( + response.status_code, + status.HTTP_400_BAD_REQUEST) + self.assertIn( + 'entities', + content_data.keys()) + entities_errors = content_data.get('entities') + self.assertIn( + 'label', + entities_errors[0]) + def test_intent_or_entity_required(self): response, content_data = self.request( self.owner_token, diff --git a/bothub/api/validators.py b/bothub/api/validators.py index 919d28550..125ed61af 100644 --- a/bothub/api/validators.py +++ b/bothub/api/validators.py @@ -82,3 +82,13 @@ def __call__(self, attrs): if not intent and not entities: raise ValidationError(_('Define a intent or one entity')) + + +class EntityNotEqualLabelValidator(object): + def __call__(self, attrs): + entity = attrs.get('entity') + label = attrs.get('label') + + if entity == label: + raise ValidationError({'label': _( + 'Label name can\'t be equal to entity name')}) From 9edba5be62cd0f6c1abe0149bd6279ee2f601bd9 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Wed, 15 Aug 2018 14:46:37 -0300 Subject: [PATCH 18/19] Add language field in new example --- bothub/api/serializers/example.py | 20 +++++++++++++++----- bothub/api/tests/test_example.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/bothub/api/serializers/example.py b/bothub/api/serializers/example.py index cff4bd28e..d83f6c440 100644 --- a/bothub/api/serializers/example.py +++ b/bothub/api/serializers/example.py @@ -7,6 +7,7 @@ from bothub.common.models import RepositoryExample from bothub.common.models import RepositoryExampleEntity from bothub.common.models import RepositoryEntity +from bothub.common import languages from ..fields import EntityText from ..fields import EntityValueField @@ -126,8 +127,9 @@ class Meta: 'repository', 'repository_update', 'text', - 'entities', + 'language', 'intent', + 'entities', ] id = serializers.PrimaryKeyRelatedField( @@ -139,11 +141,15 @@ class Meta: validators=[ CanContributeInRepositoryValidator(), ], - source='repository_update', + write_only=True, style={'show': False}) repository_update = serializers.PrimaryKeyRelatedField( read_only=True, style={'show': False}) + language = serializers.ChoiceField( + languages.LANGUAGE_CHOICES, + allow_blank=True, + required=False) entities = NewRepositoryExampleEntitySerializer( many=True, style={'text_field': 'text'}) @@ -152,11 +158,15 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.validators.append(ExampleWithIntentOrEntityValidator()) - def validate_repository(self, repository): - return repository.current_update() - def create(self, validated_data): entities_data = validated_data.pop('entities') + repository = validated_data.pop('repository') + try: + language = validated_data.pop('language') + except KeyError as e: + language = None + repository_update = repository.current_update(language or None) + validated_data.update({'repository_update': repository_update}) example = self.Meta.model.objects.create(**validated_data) for entity_data in entities_data: entity_data.update({'repository_example': example.pk}) diff --git a/bothub/api/tests/test_example.py b/bothub/api/tests/test_example.py index b034d4842..ad26123e5 100644 --- a/bothub/api/tests/test_example.py +++ b/bothub/api/tests/test_example.py @@ -9,6 +9,7 @@ from bothub.common.models import Repository from bothub.common.models import RepositoryExample from bothub.common.models import RepositoryExampleEntity +from bothub.common.models import RepositoryUpdate from ..views import NewRepositoryExampleViewSet from ..views import RepositoryExampleViewSet @@ -66,6 +67,33 @@ def test_okay(self): content_data.get('intent'), intent) + def test_okay_with_language(self): + text = 'hi' + intent = 'greet' + language = languages.LANGUAGE_PT + response, content_data = self.request( + self.owner_token, + { + 'repository': str(self.repository.uuid), + 'text': text, + 'language': language, + 'intent': intent, + 'entities': [], + }) + self.assertEqual( + response.status_code, + status.HTTP_201_CREATED) + self.assertEqual( + content_data.get('text'), + text) + self.assertEqual( + content_data.get('intent'), + intent) + repository_update_pk = content_data.get('repository_update') + repository_update = RepositoryUpdate.objects.get( + pk=repository_update_pk) + self.assertEqual(repository_update.language, language) + def test_forbidden(self): response, content_data = self.request( self.user_token, From a6f02588570674c8adc3b374042dc043e0c40efa Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Thu, 16 Aug 2018 14:58:41 -0300 Subject: [PATCH 19/19] Update repository serializer --- bothub/api/serializers/example.py | 16 +++++++++++++ bothub/api/serializers/repository.py | 8 +++++++ bothub/common/models.py | 34 ++++++++++++++++++++++++---- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/bothub/api/serializers/example.py b/bothub/api/serializers/example.py index d83f6c440..734246772 100644 --- a/bothub/api/serializers/example.py +++ b/bothub/api/serializers/example.py @@ -7,6 +7,7 @@ from bothub.common.models import RepositoryExample from bothub.common.models import RepositoryExampleEntity from bothub.common.models import RepositoryEntity +from bothub.common.models import RepositoryEntityLabel from bothub.common import languages from ..fields import EntityText @@ -192,3 +193,18 @@ def get_label(self, obj): if not obj.label: return None return obj.label.value + + +class RepositoryEntityLabelSerializer(serializers.ModelSerializer): + class Meta: + model = RepositoryEntityLabel + fields = [ + 'repository', + 'value', + 'entities', + ] + + entities = serializers.SlugRelatedField( + many=True, + slug_field='value', + read_only=True) diff --git a/bothub/api/serializers/repository.py b/bothub/api/serializers/repository.py index 7120f1f19..112e39238 100644 --- a/bothub/api/serializers/repository.py +++ b/bothub/api/serializers/repository.py @@ -14,6 +14,7 @@ from .category import RepositoryCategorySerializer from .request import RequestRepositoryAuthorizationSerializer +from .example import RepositoryEntityLabelSerializer class NewRepositorySerializer(serializers.ModelSerializer): @@ -70,6 +71,8 @@ class Meta: 'is_private', 'intents', 'entities', + 'labels', + 'labels_list', 'examples__count', 'authorization', 'available_request_authorization', @@ -88,6 +91,8 @@ class Meta: read_only=True) categories_list = serializers.SerializerMethodField() entities = serializers.SerializerMethodField() + labels = RepositoryEntityLabelSerializer(many=True) + labels_list = serializers.SerializerMethodField() authorization = serializers.SerializerMethodField() examples__count = serializers.SerializerMethodField() request_authorization = serializers.SerializerMethodField() @@ -99,6 +104,9 @@ def get_categories_list(self, obj): def get_entities(self, obj): return obj.entities_list + def get_labels_list(self, obj): + return obj.labels_list + def get_authorization(self, obj): request = self.context.get('request') if not request: diff --git a/bothub/common/models.py b/bothub/common/models.py index 639c20f78..c23c9bdfa 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -190,14 +190,38 @@ def votes_sum(self): @property def intents(self): return list(set(self.examples( - deleted=False).exclude( + exclude_deleted=False).exclude( intent='').values_list( 'intent', flat=True))) + @property + def current_entities(self): + return self.entities.filter(value__in=self.examples( + exclude_deleted=True).exclude( + entities__entity__value__isnull=True).values_list( + 'entities__entity__value', + flat=True).distinct()) + @property def entities_list(self): - return list(set(self.entities.all().values_list('value', flat=True))) + return self.current_entities.values_list( + 'value', + flat=True).distinct() + + @property + def current_labels(self): + return self.labels.filter(entities__value__in=self.examples( + exclude_deleted=True).exclude( + entities__entity__value__isnull=True).values_list( + 'entities__entity__value', + flat=True).distinct()) + + @property + def labels_list(self): + return self.current_labels.values_list( + 'value', + flat=True).distinct() @property def admins(self): @@ -207,7 +231,7 @@ def admins(self): ] return list(set(admins)) - def examples(self, language=None, deleted=True, queryset=None): + def examples(self, language=None, exclude_deleted=True, queryset=None): if queryset is None: queryset = RepositoryExample.objects query = queryset.filter( @@ -215,7 +239,7 @@ def examples(self, language=None, deleted=True, queryset=None): if language: query = query.filter( repository_update__language=language) - if deleted: + if exclude_deleted: return query.exclude(deleted_in__isnull=False) return query @@ -323,7 +347,7 @@ class Meta: @property def examples(self): - examples = self.repository.examples(deleted=False).filter( + examples = self.repository.examples(exclude_deleted=False).filter( models.Q(repository_update__language=self.language) | models.Q(translations__language=self.language)) if self.training_started_at: