diff --git a/bothub/api/v2/evaluate/serializers.py b/bothub/api/v2/evaluate/serializers.py index 6533422f..58ee0257 100644 --- a/bothub/api/v2/evaluate/serializers.py +++ b/bothub/api/v2/evaluate/serializers.py @@ -44,7 +44,7 @@ class Meta: "entities", "created_at", ] - read_only_fields = ["deleted_in", "created_at"] + read_only_fields = ["created_at"] ref_name = None entities = RepositoryEvaluateEntitySerializer(many=True, required=False) diff --git a/bothub/api/v2/example/serializers.py b/bothub/api/v2/example/serializers.py index ce621a26..9d7b4d2a 100644 --- a/bothub/api/v2/example/serializers.py +++ b/bothub/api/v2/example/serializers.py @@ -64,7 +64,6 @@ class Meta: fields = [ "id", "repository_version", - "deleted_in", "text", "intent", "language", @@ -72,7 +71,7 @@ class Meta: "entities", "translations", ] - read_only_fields = ["deleted_in", "translations"] + read_only_fields = ["translations"] ref_name = None entities = RepositoryExampleEntitySerializer(many=True, read_only=True) diff --git a/bothub/api/v2/repository/filters.py b/bothub/api/v2/repository/filters.py index ef6b53c2..6a25d157 100644 --- a/bothub/api/v2/repository/filters.py +++ b/bothub/api/v2/repository/filters.py @@ -8,8 +8,6 @@ from bothub.common.models import RepositoryAuthorization from bothub.common.models import RequestRepositoryAuthorization -# from bothub.common.models import RepositoryUpdate - class RepositoriesFilter(filters.FilterSet): class Meta: diff --git a/bothub/api/v2/repository/serializers.py b/bothub/api/v2/repository/serializers.py index a5a47bd4..d0cb8f61 100644 --- a/bothub/api/v2/repository/serializers.py +++ b/bothub/api/v2/repository/serializers.py @@ -482,7 +482,6 @@ class Meta: fields = [ "id", "repository", - "deleted_in", "text", "intent", "language", @@ -491,7 +490,7 @@ class Meta: "translations", "repository_version", ] - read_only_fields = ["deleted_in"] + read_only_fields = [] ref_name = None id = serializers.PrimaryKeyRelatedField(read_only=True, style={"show": False}) @@ -547,7 +546,6 @@ def create(self, validated_data): intent=validated_data.get("intent"), repository_version_language__repository_version__repository=repository, repository_version_language__language=language, - deleted_in__isnull=True, ): raise APIExceptionCustom( detail=_("Intention and Sentence already exists") @@ -560,7 +558,6 @@ def create(self, validated_data): intent=validated_data.get("intent"), repository_version_language=repository_version_language, repository_version_language__repository_version__is_default=True, - deleted_in__isnull=True, repository_version_language__language=language, ): raise APIExceptionCustom( diff --git a/bothub/api/v2/repository/views.py b/bothub/api/v2/repository/views.py index 55aac5f3..96612410 100644 --- a/bothub/api/v2/repository/views.py +++ b/bothub/api/v2/repository/views.py @@ -492,8 +492,3 @@ def upload_examples(self, request, **kwargs): not_added.append(data) return Response({"added": count_added, "not_added": not_added}) - - def perform_destroy(self, obj): - if obj.deleted_in: - raise APIException(_("Example already deleted")) - obj.delete() diff --git a/bothub/api/v2/tests/test_repository.py b/bothub/api/v2/tests/test_repository.py index 4adaa2ff..b70d95da 100644 --- a/bothub/api/v2/tests/test_repository.py +++ b/bothub/api/v2/tests/test_repository.py @@ -18,6 +18,7 @@ from bothub.api.v2.repository.views import RepositoryVotesViewSet from bothub.api.v2.repository.views import SearchRepositoriesViewSet from bothub.api.v2.tests.utils import create_user_and_token +from bothub.api.v2.versionning.views import RepositoryVersionViewSet from bothub.common import languages from bothub.common.models import Repository from bothub.common.models import RepositoryAuthorization @@ -1276,7 +1277,7 @@ def test_already_deleted(self): self.example.delete() response = self.request(self.example, self.owner_token) - self.assertEqual(response.status_code, status.HTTP_500_INTERNAL_SERVER_ERROR) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) class RepositoryExampleUpdateTestCase(TestCase): @@ -1781,3 +1782,56 @@ def test_text_required(self): ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertIn("text", content_data.keys()) + + +class VersionsTestCase(TestCase): + def setUp(self): + self.factory = RequestFactory() + + self.owner, self.owner_token = create_user_and_token("owner") + + self.repository = Repository.objects.create( + owner=self.owner, + name="Testing", + slug="test", + language=languages.LANGUAGE_EN, + ) + current_version = self.repository.current_version() + RepositoryExample.objects.create( + repository_version_language=current_version, + text="my name is Douglas", + intent="greet", + ) + RepositoryExample.objects.create( + repository_version_language=current_version, + text="my name is John", + intent="greet", + ) + current_version.start_training(self.owner) + + def request(self, data, token=None): + authorization_header = ( + {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} if token else {} + ) + request = self.factory.get( + "/v2/repository/version/", data, **authorization_header + ) + response = RepositoryVersionViewSet.as_view({"get": "list"})(request) + response.render() + content_data = json.loads(response.content) + return (response, content_data) + + def test_okay(self): + response, content_data = self.request( + {"repository": str(self.repository.uuid)}, self.owner_token + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(content_data.get("count"), 1) + + def test_not_authenticated(self): + response, content_data = self.request({"repository": str(self.repository.uuid)}) + self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + + def test_without_repository(self): + response, content_data = self.request({}, self.owner_token) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) diff --git a/bothub/api/v2/versionning/serializers.py b/bothub/api/v2/versionning/serializers.py index 1a16de8b..d671f763 100644 --- a/bothub/api/v2/versionning/serializers.py +++ b/bothub/api/v2/versionning/serializers.py @@ -84,23 +84,13 @@ def create(self, validated_data): ) for example in examples: - if example.deleted_in: - example_id = RepositoryExample.objects.create( - repository_version_language=version_language, - deleted_in=instance, - text=example.text, - intent=example.intent, - created_at=example.created_at, - last_update=example.last_update, - ) - else: - example_id = RepositoryExample.objects.create( - repository_version_language=version_language, - text=example.text, - intent=example.intent, - created_at=example.created_at, - last_update=example.last_update, - ) + example_id = RepositoryExample.objects.create( + repository_version_language=version_language, + text=example.text, + intent=example.intent, + created_at=example.created_at, + last_update=example.last_update, + ) example_entites = RepositoryExampleEntity.objects.filter( repository_example=example diff --git a/bothub/common/migrations/0041_delete_examples_isdeleted.py b/bothub/common/migrations/0041_delete_examples_isdeleted.py new file mode 100644 index 00000000..8637fcf3 --- /dev/null +++ b/bothub/common/migrations/0041_delete_examples_isdeleted.py @@ -0,0 +1,17 @@ +from django.db import migrations + + +def noop(apps, schema_editor): # pragma: no cover + pass + + +def delete_examples_already_deleted(apps, schema_editor): # pragma: no cover + RepositoryExample = apps.get_model("common", "RepositoryExample") + RepositoryExample.objects.filter(deleted_in__isnull=False).delete() + + +class Migration(migrations.Migration): + + dependencies = [("common", "0040_initial")] + + operations = [migrations.RunPython(delete_examples_already_deleted, noop)] diff --git a/bothub/common/migrations/0041_auto_20191212_1714.py b/bothub/common/migrations/0042_auto_20191212_2013.py similarity index 94% rename from bothub/common/migrations/0041_auto_20191212_1714.py rename to bothub/common/migrations/0042_auto_20191212_2013.py index 464827e4..1339f10d 100644 --- a/bothub/common/migrations/0041_auto_20191212_1714.py +++ b/bothub/common/migrations/0042_auto_20191212_2013.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.11 on 2019-12-12 17:14 +# Generated by Django 2.1.11 on 2019-12-12 20:13 import bothub.common.languages from django.conf import settings @@ -11,7 +11,7 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ("common", "0040_initial"), + ("common", "0041_delete_examples_isdeleted"), ] operations = [ @@ -122,7 +122,7 @@ class Migration(migrations.Migration): ), ( "last_update", - models.DateTimeField(auto_now_add=True, verbose_name="last update"), + models.DateTimeField(null=True, verbose_name="last update"), ), ( "total_training_end", @@ -151,6 +151,7 @@ class Migration(migrations.Migration): migrations.RemoveField( model_name="repositoryevaluateresult", name="repository_update" ), + migrations.RemoveField(model_name="repositoryexample", name="deleted_in"), migrations.RemoveField( model_name="repositoryexample", name="repository_update" ), @@ -176,17 +177,6 @@ class Migration(migrations.Migration): to="common.RepositoryVersionLanguage", ), ), - migrations.AlterField( - model_name="repositoryexample", - name="deleted_in", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - related_name="deleted", - to="common.RepositoryVersionLanguage", - ), - ), migrations.DeleteModel(name="RepositoryUpdate"), migrations.AddField( model_name="repositoryversion", diff --git a/bothub/common/models.py b/bothub/common/models.py index 8bf8c966..c18669d7 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -63,7 +63,7 @@ def order_by_relevance(self): ) def supported_language(self, language): - valid_examples = RepositoryExample.objects.filter(deleted_in__isnull=True) + valid_examples = RepositoryExample.objects.all() valid_updates = RepositoryVersionLanguage.objects.filter( added__in=valid_examples ) @@ -402,8 +402,6 @@ def examples( ) if language: query = query.filter(repository_version_language__language=language) - if exclude_deleted: - return query.exclude(deleted_in__isnull=False) return query def evaluations( @@ -420,8 +418,6 @@ def evaluations( ) if language: query = query.filter(repository_version_language__language=language) - if exclude_deleted: - return query.exclude(deleted_in__isnull=False) return query # pragma: no cover def evaluations_results(self, queryset=None, version_default=True): @@ -478,10 +474,6 @@ def current_version(self, language=None, is_default=True): repository_version, created = self.versions.get_or_create(is_default=is_default) - if repository_version.created_by is None: - repository_version.created_by = repository_version.repository.owner - repository_version.save(update_fields=["created_by"]) - repository_version_language, created = RepositoryVersionLanguage.objects.get_or_create( repository_version=repository_version, language=language ) @@ -566,12 +558,10 @@ class Meta: choices=Repository.ALGORITHM_CHOICES, default=Repository.ALGORITHM_STATISTICAL_MODEL, ) - repository_version = models.ForeignKey( - RepositoryVersion, models.CASCADE - ) # updates related_name + repository_version = models.ForeignKey(RepositoryVersion, models.CASCADE) training_log = models.TextField(_("training log"), blank=True, editable=False) created_at = models.DateTimeField(_("created at"), auto_now_add=True) - last_update = models.DateTimeField(_("last update"), auto_now_add=True) + last_update = models.DateTimeField(_("last update"), null=True) total_training_end = models.IntegerField( _("total training end"), default=0, blank=False, null=False ) @@ -585,15 +575,7 @@ def examples(self): | models.Q(translations__language=self.language) ) if self.training_end_at and (self.training_end_at >= self.last_update): - t_started_at = self.training_started_at - examples = examples.exclude( - models.Q(last_update__lte=self.training_end_at) - | models.Q(deleted_in=self) - | models.Q(deleted_in__training_started_at__lt=t_started_at) - ) - - else: - examples = examples.exclude(deleted_in__isnull=False) + examples = examples.exclude(models.Q(last_update__lte=self.training_end_at)) return examples.distinct() @property @@ -695,11 +677,7 @@ def ready_for_train(self): if previous_update.failed_at: return True - if ( - not self.added.exists() - and not self.translated_added.exists() - and not self.deleted.exists() - ): + if not self.added.exists() and not self.translated_added.exists(): return False if self.examples.count() == 0: @@ -731,10 +709,6 @@ def __str__(self): return "Repository Version Language #{}".format(self.id) # pragma: no cover def validate_init_train(self, by=None): - # if self.trained_at: - # raise RepositoryUpdateAlreadyTrained() - # if self.training_started_at: - # raise RepositoryUpdateAlreadyStartedTraining() if by: authorization = self.repository_version.repository.get_user_authorization( by @@ -764,13 +738,20 @@ def start_training(self, created_by): self.repository_version.save(update_fields=["created_by"]) def save_training(self, bot_data): - # if self.trained_at: - # raise RepositoryUpdateAlreadyTrained() + last_time = timezone.now() - self.training_end_at = timezone.now() + self.training_end_at = last_time + self.last_update = last_time self.bot_data = bot_data self.total_training_end += 1 - self.save(update_fields=["total_training_end", "training_end_at", "bot_data"]) + self.save( + update_fields=[ + "total_training_end", + "training_end_at", + "bot_data", + "last_update", + ] + ) def get_bot_data(self): return self.bot_data @@ -789,13 +770,6 @@ class Meta: repository_version_language = models.ForeignKey( RepositoryVersionLanguage, models.CASCADE, related_name="added", editable=False ) - deleted_in = models.ForeignKey( - RepositoryVersionLanguage, - models.CASCADE, - related_name="deleted", - blank=True, - null=True, - ) text = models.TextField(_("text"), help_text=_("Example text")) intent = models.CharField( _("intent"), @@ -838,11 +812,10 @@ def get_entities(self, language): # pragma: no cover return self.entities.all() return self.get_translation(language).entities.all() - def delete(self): - self.deleted_in = self.repository_version_language.repository_version.repository.current_version( - self.repository_version_language.language - ) - self.save(update_fields=["deleted_in"]) + def delete(self, using=None, keep_parents=False): + self.repository_version_language.last_update = timezone.now() + self.repository_version_language.save(update_fields=["last_update"]) + return super().delete(using, keep_parents) class RepositoryTranslatedExampleManager(models.Manager): diff --git a/bothub/common/tests.py b/bothub/common/tests.py index b44f9fa6..19899234 100644 --- a/bothub/common/tests.py +++ b/bothub/common/tests.py @@ -1,6 +1,7 @@ from django.conf import settings from django.core.exceptions import ValidationError from django.test import TestCase +from django.utils import timezone from bothub.authentication.models import User from . import languages @@ -17,7 +18,7 @@ from .models import RequestRepositoryAuthorization -class RepositoryUpdateTestCase(TestCase): +class RepositoryVersionTestCase(TestCase): def setUp(self): owner = User.objects.create_user("fake@user.com", "user", "123456") self.repository = Repository.objects.create(owner=owner, slug="test") @@ -32,12 +33,12 @@ def setUp(self): def test_repository_example_entity(self): self.assertEqual(self.entity.value, "User") - # def test_repository_current_update(self): - # update1 = self.repository.current_version("en") - # self.assertEqual(update1, self.repository.current_version("en")) - # update1.training_started_at = timezone.now() - # update1.save() - # self.assertNotEqual(update1, self.repository.current_version("en")) + def test_repository_current_version(self): + update1 = self.repository.current_version("en") + self.assertEqual(update1, self.repository.current_version("en")) + update1.training_started_at = timezone.now() + update1.save() + self.assertEqual(update1, self.repository.current_version("en")) class TranslateTestCase(TestCase): @@ -294,10 +295,6 @@ def test_language(self): ) self.assertEqual(self.example.language, self.language) - def test_delete(self): - self.example.delete() - self.assertEqual(self.example.deleted_in, self.repository.current_version()) - class RepositoryAuthorizationTestCase(TestCase): def setUp(self): @@ -439,7 +436,7 @@ def test_role_contributor_can_contribute(self): self.assertTrue(authorization_user.can_contribute) -class RepositoryUpdateTrainingTestCase(TestCase): +class RepositoryVersionTrainingTestCase(TestCase): def setUp(self): self.owner = User.objects.create_user("owner@user.com", "user") @@ -456,23 +453,6 @@ def test_train(self): update.save_training(bot_data) self.assertEqual(update.get_bot_data(), bot_data) - # def test_already_started_trained(self): - # update = self.repository.current_version() - # update.start_training(self.owner) - # with self.assertRaises(RepositoryUpdateAlreadyStartedTraining): - # update.start_training(self.owner) - - # def test_already_trained(self): - # update = self.repository.current_version() - # update.start_training(self.owner) - # update.save_training(b"") - # - # with self.assertRaises(RepositoryUpdateAlreadyTrained): - # update.start_training(self.owner) - # - # with self.assertRaises(RepositoryUpdateAlreadyTrained): - # update.save_training(self.owner) - def test_training_not_allowed(self): user = User.objects.create_user("fake@user.com", "fake") @@ -481,7 +461,7 @@ def test_training_not_allowed(self): update.start_training(user) -class RepositoryUpdateExamplesTestCase(TestCase): +class RepositoryVersionExamplesTestCase(TestCase): def setUp(self): self.owner = User.objects.create_user("owner@user.com", "user") @@ -489,12 +469,12 @@ def setUp(self): owner=self.owner, name="Test", slug="test", language=languages.LANGUAGE_EN ) RepositoryExample.objects.create( - repository_update=self.repository.current_version(), + repository_version_language=self.repository.current_version(), text="hi", intent="greet", ) example = RepositoryExample.objects.create( - repository_update=self.repository.current_version(), + repository_version_language=self.repository.current_version(), text="hello1", intent="greet", ) @@ -504,70 +484,22 @@ def setUp(self): self.update.start_training(self.owner) self.update.save_training(b"") - # def test_okay(self): - # new_update_1 = self.repository.current_version() - # RepositoryExample.objects.create( - # repository_update=new_update_1, text="hello", intent="greet" - # ) - # new_update_1.start_training(self.owner) - # - # new_update_2 = self.repository.current_version() - # RepositoryExample.objects.create( - # repository_update=new_update_2, text="good morning", intent="greet" - # ) - # - # self.assertEqual(self.update.examples.count(), 1) - # self.assertEqual(new_update_1.examples.count(), 2) - # self.assertEqual(new_update_2.examples.count(), 3) - - # def test_examples_deleted_consistency(self): - # new_update_1 = self.repository.current_version() - # RepositoryExample.objects.create( - # repository_update=new_update_1, text="hello", intent="greet" - # ) - # RepositoryExample.objects.create( - # repository_update=new_update_1, text="hello d1", intent="greet" - # ).delete() - # examples_1_count = new_update_1.examples.count() - # new_update_1.start_training(self.owner) - # - # new_update_2 = self.repository.current_version() - # RepositoryExample.objects.create( - # repository_update=new_update_2, text="hellow", intent="greet" - # ) - # examples_2_count = new_update_2.examples.count() - # new_update_2.start_training(self.owner) - # - # new_update_3 = self.repository.current_version() - # RepositoryExample.objects.create( - # repository_update=new_update_3, text="hellow", intent="greet" - # ) - # RepositoryExample.objects.create( - # repository_update=new_update_3, text="hello d2", intent="greet" - # ).delete() - # RepositoryExample.objects.create( - # repository_update=new_update_3, text="hello d3", intent="greet" - # ).delete() - # RepositoryExample.objects.create( - # repository_update=new_update_3, text="hello d4", intent="greet" - # ).delete() - # examples_3_count = new_update_3.examples.count() - # new_update_3.start_training(self.owner) - # - # new_update_4 = self.repository.current_version() - # RepositoryExample.objects.create( - # repository_update=new_update_4, text="hellow", intent="greet" - # ) - # examples_4_count = new_update_4.examples.count() - # new_update_4.start_training(self.owner) - # - # self.assertEqual(examples_1_count, new_update_1.examples.count()) - # - # self.assertEqual(examples_2_count, new_update_2.examples.count()) - # - # self.assertEqual(examples_3_count, new_update_3.examples.count()) - # - # self.assertEqual(examples_4_count, new_update_4.examples.count()) + def test_okay(self): + new_update_1 = self.repository.current_version() + RepositoryExample.objects.create( + repository_version_language=new_update_1, text="hello", intent="greet" + ) + new_update_1.start_training(self.owner) + + new_update_2 = self.repository.current_version() + RepositoryExample.objects.create( + repository_version_language=new_update_2, + text="good morning", + intent="greet", + ) + self.assertEqual(self.update.examples.count(), 2) + self.assertEqual(new_update_1.examples.count(), 3) + self.assertEqual(new_update_2.examples.count(), 3) class RepositoryReadyForTrain(TestCase):