Merge b5eb0e4 into e2a9748

README.md

@@ -32,6 +32,16 @@ Use ```make``` commands to ```check_environment```, ```install_requirements```,
 Run ```pipenv run python ./manage.py fill_db_using_fake_data``` to fill database using fake data. This can help you to test [Bothub Webapp](https://github.com/Ilhasoft/bothub-webapp).
 
 
+### Update Repositories Tests
+
+Run ```pipenv run python ./manage.py update_repository_tests``` update repositories to sort by total tests.
+
+
+### Migrate all training for aws
+
+Run ```pipenv run python ./manage.py transfer_train_aws``` Migrate all trainings to an aws bucket defined in project settings.
+
+
 #### Fake users infos:
 
 | nickname | email | password | is superuser |

bothub/api/v2/nlp/views.py

@@ -100,6 +100,7 @@ def start_training(self, request, **kwargs):
                 "algorithm": repository.algorithm,
                 "use_name_entities": repository.use_name_entities,
                 "use_competing_intents": repository.use_competing_intents,
+                "use_analyze_char": repository.use_analyze_char,
                 "ALGORITHM_STATISTICAL_MODEL": Repository.ALGORITHM_STATISTICAL_MODEL,
                 "ALGORITHM_NEURAL_NETWORK_EXTERNAL": Repository.ALGORITHM_NEURAL_NETWORK_EXTERNAL,
             }

bothub/api/v2/repository/serializers.py

@@ -193,6 +193,7 @@ class Meta:
             "use_language_model_featurizer",
             "use_competing_intents",
             "use_name_entities",
+            "use_analyze_char",
             "nlp_server",
         ]
         read_only = [

bothub/common/admin.py

@@ -17,6 +17,7 @@ class RepositoryUpdateInline(admin.TabularInline):
         "algorithm",
         "use_competing_intents",
         "use_name_entities",
+        "use_analyze_char",
         "created_at",
         "by",
         "training_started_at",

bothub/common/management/commands/transfer_train_aws.py

@@ -0,0 +1,18 @@
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from bothub.common.models import RepositoryUpdate
+from bothub.utils import send_bot_data_file_aws
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **kwargs):
+        if settings.AWS_SEND:
+            for update in RepositoryUpdate.objects.all().exclude(bot_data__exact=""):
+                repository_update = RepositoryUpdate.objects.get(pk=update.pk)
+                bot_data = send_bot_data_file_aws(update.pk, update.bot_data)
+                repository_update.bot_data = bot_data
+                repository_update.save(update_fields=["bot_data"])
+                print("Updating bot_data repository_update {}".format(str(update.pk)))
+        else:
+            print("You need to configure the environment variables for AWS.")

bothub/common/management/commands/update_repository_tests.py

@@ -0,0 +1,15 @@
+from django.core.management.base import BaseCommand
+
+from bothub.common.models import Repository
+from bothub.common.models import RepositoryUpdate
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **kwargs):
+        print("Updating...")
+        Repository.objects.all().update(total_updates=0)
+        for update in RepositoryUpdate.objects.all().filter(trained_at__isnull=False):
+            repository = Repository.objects.get(uuid=update.repository.uuid)
+            repository.total_updates += 1
+            repository.save()
+            print("Repository UUID {} Updated".format(repository.pk))

bothub/common/migrations/0032_repository_total_updates.py

@@ -6,13 +6,6 @@
 from bothub.common.models import Repository
 
 
-def updateRepository(apps, schema_editor):
-    for update in RepositoryUpdate.objects.all().filter(trained_at__isnull=False):
-        repository = Repository.objects.get(uuid=update.repository.uuid)
-        repository.total_updates += 1
-        repository.save()
-
-
 class Migration(migrations.Migration):
 
     dependencies = [("common", "0031_auto_20190502_1732")]
@@ -22,6 +15,5 @@ class Migration(migrations.Migration):
             model_name="repository",
             name="total_updates",
             field=models.IntegerField(default=0, verbose_name="total updates"),
-        ),
-        migrations.RunPython(updateRepository),
+        )
     ]

bothub/common/migrations/0037_auto_20191011_2021.py

@@ -2,19 +2,6 @@
 
 import django.utils.timezone
 from django.db import migrations, models
-from django.conf import settings
-from bothub.utils import send_bot_data_file_aws
-from bothub.common.models import RepositoryUpdate
-
-
-def update_repository(apps, schema_editor):
-    if settings.AWS_SEND:
-        for update in RepositoryUpdate.objects.all().exclude(bot_data__exact=""):
-            repository_update = RepositoryUpdate.objects.get(pk=update.pk)
-            bot_data = send_bot_data_file_aws(update.pk, update.bot_data)
-            repository_update.bot_data = bot_data
-            repository_update.save(update_fields=["bot_data"])
-            print("Updating bot_data repository_update {}".format(str(update.pk)))
 
 
 class Migration(migrations.Migration):
@@ -35,7 +22,6 @@ class Migration(migrations.Migration):
                 default=django.utils.timezone.now, editable=False
             ),
         ),
-        migrations.RunPython(update_repository),
         migrations.AlterField(
             model_name="repositoryupdate",
             name="bot_data",

bothub/common/migrations/0038_auto_20191101_1935.py

@@ -0,0 +1,21 @@
+# Generated by Django 2.1.11 on 2019-11-01 19:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("common", "0037_auto_20191011_2021")]
+
+    operations = [
+        migrations.AddField(
+            model_name="repository",
+            name="use_analyze_char",
+            field=models.BooleanField(default=False, verbose_name="Use analyze char"),
+        ),
+        migrations.AddField(
+            model_name="repositoryupdate",
+            name="use_analyze_char",
+            field=models.BooleanField(default=False),
+        ),
+    ]

bothub/common/models.py

@@ -148,6 +148,7 @@ class Meta:
         ),
         default=False,
     )
+    use_analyze_char = models.BooleanField(_("Use analyze char"), default=False)
     categories = models.ManyToManyField(
         RepositoryCategory, help_text=CATEGORIES_HELP_TEXT
     )
@@ -461,6 +462,7 @@ class Meta:
     trained_at = models.DateTimeField(_("trained at"), blank=True, null=True)
     failed_at = models.DateTimeField(_("failed at"), blank=True, null=True)
     training_log = models.TextField(_("training log"), blank=True, editable=False)
+    use_analyze_char = models.BooleanField(default=False)
 
     @property
     def examples(self):
@@ -559,6 +561,8 @@ def ready_for_train(self):
                 is not self.repository.use_name_entities
             ):
                 return True
+            if previous_update.use_analyze_char is not self.repository.use_analyze_char:
+                return True
             if previous_update.failed_at:
                 return True
 
@@ -614,13 +618,15 @@ def start_training(self, by):
         self.algorithm = self.repository.algorithm
         self.use_competing_intents = self.repository.use_competing_intents
         self.use_name_entities = self.repository.use_name_entities
+        self.use_analyze_char = self.repository.use_analyze_char
         self.save(
             update_fields=[
                 "by",
                 "training_started_at",
                 "algorithm",
                 "use_competing_intents",
                 "use_name_entities",
+                "use_analyze_char",
             ]
         )