Merge 943ff68 into 4ff188b

README.md

@@ -88,3 +88,4 @@ You can set environment variables in your OS, write on ```.env``` file or pass v
 | BOTHUB_ENGINE_AWS_ACCESS_KEY_ID | ```string``` | ```None``` | 
 | BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY | ```string``` | ```None``` | 
 | BOTHUB_ENGINE_AWS_REGION_NAME | ```string``` | ```None``` | 
+| BOTHUB_ENGINE_AWS_SEND |  ```bool``` | ```False``` | 

bothub/api/v2/nlp/views.py

@@ -2,9 +2,12 @@
 import json
 import requests
 
+from django.conf import settings
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 from django.shortcuts import get_object_or_404
+from django.core.validators import URLValidator
+from django.core.exceptions import ValidationError
 
 from rest_framework import mixins
 from rest_framework import exceptions
@@ -417,11 +420,23 @@ class RepositoryUpdateInterpretersViewSet(
     def retrieve(self, request, *args, **kwargs):
         check_auth(request)
         update = self.get_object()
+
+        validator = URLValidator()
+
         try:
-            download = requests.get(update.bot_data)
-            bot_data = base64.b64encode(download.content)
-        except Exception:
-            bot_data = b""
+            validator(str(update.bot_data))
+            url_valid = True
+        except ValidationError:
+            url_valid = False
+
+        if url_valid:
+            try:
+                download = requests.get(update.bot_data)
+                bot_data = base64.b64encode(download.content)
+            except Exception:
+                bot_data = b""
+        else:
+            bot_data = update.bot_data
         return Response(
             {
                 "update_id": update.id,
@@ -434,6 +449,9 @@ def create(self, request, *args, **kwargs):
         check_auth(request)
         id = request.data.get("id")
         repository = get_object_or_404(RepositoryUpdate, pk=id)
-        bot_data = base64.b64decode(request.data.get("bot_data"))
-        repository.save_training(send_bot_data_file_aws(id, bot_data))
+        if settings.AWS_SEND:
+            bot_data = base64.b64decode(request.data.get("bot_data"))
+            repository.save_training(send_bot_data_file_aws(id, bot_data))
+        else:
+            repository.save_training(request.data.get("bot_data"))
         return Response({})

bothub/common/migrations/0035_auto_20190902_1455.py

@@ -1,17 +1,18 @@
 # Generated by Django 2.1.5 on 2019-09-02 14:55
-
+from django.conf import settings
 from django.db import migrations, models
 from bothub.utils import send_bot_data_file_aws
 from bothub.common.models import RepositoryUpdate
 
 
 def update_repository(apps, schema_editor):
-    for update in RepositoryUpdate.objects.all().exclude(bot_data__exact=""):
-        url = send_bot_data_file_aws(update.pk, update.bot_data)
-        repository_update = RepositoryUpdate.objects.get(pk=update.pk)
-        repository_update.bot_data = url
-        repository_update.save(update_fields=["bot_data"])
-        print("Updating bot_data repository_update {}".format(str(update.pk)))
+    if settings.AWS_SEND:
+        for update in RepositoryUpdate.objects.all().exclude(bot_data__exact=""):
+            repository_update = RepositoryUpdate.objects.get(pk=update.pk)
+            bot_data = send_bot_data_file_aws(update.pk, update.bot_data)
+            repository_update.bot_data = bot_data
+            repository_update.save(update_fields=["bot_data"])
+            print("Updating bot_data repository_update {}".format(str(update.pk)))
 
 
 class Migration(migrations.Migration):
@@ -23,6 +24,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name="repositoryupdate",
             name="bot_data",
-            field=models.URLField(blank=True, verbose_name="bot data"),
+            field=models.TextField(blank=True, verbose_name="bot data"),
         ),
     ]

bothub/common/models.py

@@ -453,7 +453,7 @@ class Meta:
     use_competing_intents = models.BooleanField(default=False)
     use_name_entities = models.BooleanField(default=False)
     created_at = models.DateTimeField(_("created at"), auto_now_add=True)
-    bot_data = models.URLField(_("bot data"), blank=True)
+    bot_data = models.TextField(_("bot data"), blank=True)
     by = models.ForeignKey(User, models.CASCADE, blank=True, null=True)
     training_started_at = models.DateTimeField(
         _("training started at"), blank=True, null=True

bothub/settings.py

@@ -44,6 +44,7 @@
     BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY=(str, ""),
     BOTHUB_ENGINE_AWS_S3_BUCKET_NAME=(str, ""),
     BOTHUB_ENGINE_AWS_REGION_NAME=(str, "us-east-1"),
+    BOTHUB_ENGINE_AWS_SEND=(bool, False),
 )
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -257,7 +258,7 @@
 
 
 # AWS
-
+AWS_SEND = env.bool("BOTHUB_ENGINE_AWS_SEND")
 AWS_ACCESS_KEY_ID = env.str("BOTHUB_ENGINE_AWS_ACCESS_KEY_ID")
 AWS_SECRET_ACCESS_KEY = env.str("BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY")
 AWS_BUCKET_NAME = env.str("BOTHUB_ENGINE_AWS_S3_BUCKET_NAME")