Merge pull request #241 from Ilhasoft/repository-add-algorithm

Add algorithm field in Repository model

Pipfile

@@ -4,7 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-django = "==2.1.3"
+django = "==2.1.5"
 dj-database-url = "==0.5.0"
 python-decouple = "==3.1"
 djangorestframework = "==3.9.0"

bothub/api/v1/serializers/repository.py

@@ -26,7 +26,7 @@ class Meta:
             'name',
             'slug',
             'language',
-            'use_language_model_featurizer',
+            'algorithm',
             'use_competing_intents',
             'categories',
             'description',
@@ -64,6 +64,7 @@ class Meta:
             'slug',
             'language',
             'available_languages',
+            'algorithm',
             'use_language_model_featurizer',
             'use_competing_intents',
             'categories',

bothub/api/v2/repository/serializers.py

@@ -113,6 +113,7 @@ class Meta:
             'request_authorization',
             'available_request_authorization',
             'languages_warnings',
+            'algorithm',
             'use_language_model_featurizer',
             'use_competing_intents',
         ]

bothub/common/admin.py

@@ -14,7 +14,7 @@ class RepositoryUpdateInline(admin.TabularInline):
 
     fields = [
         'language',
-        'use_language_model_featurizer',
+        'algorithm',
         'use_competing_intents',
         'created_at',
         'by',

bothub/common/migrations/0028_auto_20190121_1250.py

@@ -0,0 +1,47 @@
+# Generated by Django 2.1.3 on 2019-01-21 12:50
+
+from django.db import migrations, models
+
+
+def populate_algorithm(apps, *args):
+    from bothub.common.models import Repository as R
+    Repository = apps.get_model('common', 'Repository')
+    for repository in Repository.objects.all():
+        repository.algorithm = R.ALGORITHM_NEURAL_NETWORK_EXTERNAL \
+            if repository.use_language_model_featurizer else \
+            R.ALGORITHM_NEURAL_NETWORK_INTERNAL
+        repository.save(update_fields=['algorithm'])
+        for update in repository.updates.all():
+            update.algorithm = R.ALGORITHM_NEURAL_NETWORK_EXTERNAL \
+                if update.use_language_model_featurizer else \
+                R.ALGORITHM_NEURAL_NETWORK_INTERNAL
+            update.save(update_fields=['algorithm'])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0027_repositorycategory_icon'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='repository',
+            name='algorithm',
+            field=models.CharField(choices=[('statistical_model', 'Statistical Model'), ('neural_network_internal', 'Neural Network with internal vocabulary'), ('neural_network_external', 'Neural Network with external vocabulary (BETA)')], default='statistical_model', max_length=24, verbose_name='algorithm'),
+        ),
+        migrations.AddField(
+            model_name='repositoryupdate',
+            name='algorithm',
+            field=models.CharField(choices=[('statistical_model', 'Statistical Model'), ('neural_network_internal', 'Neural Network with internal vocabulary'), ('neural_network_external', 'Neural Network with external vocabulary (BETA)')], default='statistical_model', max_length=24, verbose_name='algorithm'),
+        ),
+        migrations.RunPython(populate_algorithm),
+        migrations.RemoveField(
+            model_name='repository',
+            name='use_language_model_featurizer',
+        ),
+        migrations.RemoveField(
+            model_name='repositoryupdate',
+            name='use_language_model_featurizer',
+        ),
+    ]

bothub/common/models.py

@@ -98,6 +98,24 @@ class Meta:
                              'the same purpose')
     DESCRIPTION_HELP_TEXT = _('Tell what your bot do!')
 
+    ALGORITHM_STATISTICAL_MODEL = 'statistical_model'
+    ALGORITHM_NEURAL_NETWORK_INTERNAL = 'neural_network_internal'
+    ALGORITHM_NEURAL_NETWORK_EXTERNAL = 'neural_network_external'
+    ALGORITHM_CHOICES = [
+        (
+            ALGORITHM_STATISTICAL_MODEL,
+            _('Statistical Model'),
+        ),
+        (
+            ALGORITHM_NEURAL_NETWORK_INTERNAL,
+            _('Neural Network with internal vocabulary'),
+        ),
+        (
+            ALGORITHM_NEURAL_NETWORK_EXTERNAL,
+            _('Neural Network with external vocabulary (BETA)'),
+        ),
+    ]
+
     uuid = models.UUIDField(
         _('UUID'),
         primary_key=True,
@@ -123,12 +141,12 @@ class Meta:
         validators=[
             languages.validate_language,
         ])
-    use_language_model_featurizer = models.BooleanField(
-        _('Use language model featurizer'),
-        help_text=_('You can use language featurizer to get words ' +
-                    'similarity. You need less examples to create a great ' +
-                    'bot.'),
-        default=True)
+    algorithm = models.CharField(
+        _('algorithm'),
+        max_length=24,
+        choices=ALGORITHM_CHOICES,
+        default=ALGORITHM_STATISTICAL_MODEL,
+    )
     use_competing_intents = models.BooleanField(
         _('Use competing intents'),
         help_text=_('When using competing intents the confidence of the ' +
@@ -288,6 +306,10 @@ def admins(self):
         ]
         return list(set(admins))
 
+    @property
+    def use_language_model_featurizer(self):
+        return self.algorithm != Repository.ALGORITHM_NEURAL_NETWORK_INTERNAL
+
     def __str__(self):
         return 'Repository {} - {}/{}'.format(
             self.name,
@@ -389,7 +411,12 @@ class Meta:
         validators=[
             languages.validate_language,
         ])
-    use_language_model_featurizer = models.BooleanField(default=True)
+    algorithm = models.CharField(
+        _('algorithm'),
+        max_length=24,
+        choices=Repository.ALGORITHM_CHOICES,
+        default=Repository.ALGORITHM_STATISTICAL_MODEL,
+    )
     use_competing_intents = models.BooleanField(default=False)
     created_at = models.DateTimeField(
         _('created at'),
@@ -493,8 +520,7 @@ def ready_for_train(self):
             created_at__lt=self.created_at).first()
 
         if previous_update:
-            if previous_update.use_language_model_featurizer is not \
-               self.repository.use_language_model_featurizer:
+            if previous_update.algorithm != self.repository.algorithm:
                 return True
             if previous_update.use_competing_intents is not \
                self.repository.use_competing_intents:
@@ -525,6 +551,10 @@ def warnings(self):
                            self.RECOMMENDED_INTENTS))
         return w
 
+    @property
+    def use_language_model_featurizer(self):
+        return self.algorithm != Repository.ALGORITHM_NEURAL_NETWORK_INTERNAL
+
     def __str__(self):
         return 'Repository Update #{}'.format(self.id)
 
@@ -542,14 +572,13 @@ def start_training(self, by):
         self.validate_init_train(by)
         self.by = by
         self.training_started_at = timezone.now()
-        self.use_language_model_featurizer = self.repository \
-            .use_language_model_featurizer
+        self.algorithm = self.repository.algorithm
         self.use_competing_intents = self.repository.use_competing_intents
         self.save(
             update_fields=[
                 'by',
                 'training_started_at',
-                'use_language_model_featurizer',
+                'algorithm',
                 'use_competing_intents',
             ])
 

bothub/common/tests.py

@@ -758,6 +758,22 @@ def test_last_train_failed(self):
         current_update.train_fail()
         self.assertTrue(self.repository.current_update().ready_for_train)
 
+    def test_change_algorithm(self):
+        self.assertTrue(self.repository.ready_for_train)
+        for (val_current, verb_current) in Repository.ALGORITHM_CHOICES:
+            for (val_next, verb_next) in Repository.ALGORITHM_CHOICES:
+                if val_current == val_next:
+                    continue
+                self.repository.algorithm = val_current
+                self.repository.save()
+                current_update = self.repository.current_update()
+                current_update.start_training(self.owner)
+                current_update.save_training(b'')
+                self.assertFalse(self.repository.ready_for_train)
+                self.repository.algorithm = val_next
+                self.repository.save()
+                self.assertTrue(self.repository.ready_for_train)
+
 
 class RepositoryUpdateReadyForTrain(TestCase):
     def setUp(self):
@@ -768,7 +784,7 @@ def setUp(self):
             name='Test',
             slug='test',
             language=languages.LANGUAGE_EN,
-            use_language_model_featurizer=False)
+            algorithm=Repository.ALGORITHM_NEURAL_NETWORK_INTERNAL)
 
     def test_be_true(self):
         RepositoryExample.objects.create(
@@ -874,7 +890,8 @@ def test_entity_dont_have_min_examples(self):
 
     def test_settings_change_exists_requirements(self):
         self.repository.current_update().start_training(self.owner)
-        self.repository.use_language_model_featurizer = True
+        self.repository.algorithm = Repository \
+            .ALGORITHM_NEURAL_NETWORK_EXTERNAL
         self.repository.save()
         RepositoryExample.objects.create(
             repository_update=self.repository.current_update(),
@@ -1124,7 +1141,7 @@ def setUp(self):
             name='Test',
             slug='test',
             language=self.language,
-            use_language_model_featurizer=True)
+            algorithm=Repository.ALGORITHM_NEURAL_NETWORK_EXTERNAL)
 
         RepositoryExample.objects.create(
             repository_update=self.repository.current_update(),
@@ -1135,22 +1152,10 @@ def setUp(self):
             text='my name is John',
             intent='greet')
 
-    def test_change_ready_for_train(self):
-        self.assertTrue(self.repository.ready_for_train)
-        current_update = self.repository.current_update()
-        current_update.start_training(self.owner)
-        current_update.save_training(b'')
-        self.assertFalse(self.repository.ready_for_train)
-        self.repository.use_language_model_featurizer = False
-        self.repository.save()
-        self.assertTrue(self.repository.ready_for_train)
-        self.repository.use_language_model_featurizer = True
-        self.repository.save()
-        self.assertFalse(self.repository.ready_for_train)
-
     def test_equal_repository_value_after_train(self):
         current_update = self.repository.current_update()
-        self.repository.use_language_model_featurizer = False
+        self.repository.algorithm = Repository \
+            .ALGORITHM_NEURAL_NETWORK_INTERNAL
         self.repository.save()
         current_update.start_training(self.owner)
         current_update.save_training(b'')

setup.py

@@ -3,7 +3,7 @@
 
 setup(
     name='bothub-engine',
-    version='1.18.1',
+    version='1.19.0',
     description='Bothub Engine',
     packages=find_packages(),
     install_requires=[