diff --git a/Pipfile.lock b/Pipfile.lock index 574674e8..4bf45f76 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -18,18 +18,18 @@ "default": { "boto3": { "hashes": [ - "sha256:5ff78c697d8009b9fe9808baea60660f0a07be666fb7c65fc4c11756f568124e", - "sha256:d01314496080ac82ddff3d1b2c6ad542d89bfcb100b7a0e8aaf2d6aef99775c6" + "sha256:2efd0a9647ef8fd7bf9fbbfdeb77007050258aa39914569c689bdf9cc288cdc1", + "sha256:331b82e878f524be11c10d866a68b123b1c5e892d218a8250de10bf11cb14402" ], "index": "pypi", - "version": "==1.9.239" + "version": "==1.9.244" }, "botocore": { "hashes": [ - "sha256:14ff881779776a5b08e4bb8d7e0cd6ca86e36fc80decc536d605c406fb81c993", - "sha256:5df4a8f8bb579daed58368d6c15cdd72455bfee7dc86be243e2ef4dff5ba1177" + "sha256:7b75482156ef93e7a463f80411dc76fc868cd9fb1dedf03250cec98f8459d765", + "sha256:e43e7d19f203d572f78c18a6a16cb87bbea10bc0543e0285f7b6a35d0c825bb6" ], - "version": "==1.12.239" + "version": "==1.12.244" }, "certifi": { "hashes": [ @@ -110,11 +110,11 @@ }, "drf-yasg": { "hashes": [ - "sha256:68fded2ffdf46e03f33e766184b7d8f1e1a5236f94acfd0c4ba932a57b812566", - "sha256:fcef74709ead2b365410be3d12afbfd0a6e49d1efe615a15a929da7e950bb83c" + "sha256:4cfec631880ae527a91ec7cd3241aea2f82189f59e2f089119aa687761afb227", + "sha256:504cce09035cf1bace63b84d9d778b772f86bb37d8a71ed6f723346362e633b2" ], "index": "pypi", - "version": "==1.16.1" + "version": "==1.17.0" }, "gevent": { "hashes": [ @@ -199,10 +199,10 @@ }, "jinja2": { "hashes": [ - "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", - "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" + "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f", + "sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de" ], - "version": "==2.10.1" + "version": "==2.10.3" }, "jmespath": { "hashes": [ @@ -371,10 +371,10 @@ }, "attrs": { "hashes": [ - "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", - "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + "sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2", + "sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396" ], - "version": "==19.1.0" + "version": "==19.2.0" }, "autopep8": { "hashes": [ @@ -518,11 +518,11 @@ }, "prompt-toolkit": { "hashes": [ - "sha256:11adf3389a996a6d45cc277580d0d53e8a5afd281d0c9ec71b28e6f121463780", - "sha256:2519ad1d8038fd5fc8e770362237ad0364d16a7650fb5724af6997ed5515e3c1", - "sha256:977c6583ae813a37dc1c2e1b715892461fcbdaa57f6fc62f33a528c4886c8f55" + "sha256:46642344ce457641f28fc9d1c9ca939b63dadf8df128b86f1b9860e59c73a5e4", + "sha256:e7f8af9e3d70f514373bf41aa51bc33af12a6db3f71461ea47fea985defb2c31", + "sha256:f15af68f66e664eaa559d4ac8a928111eebd5feda0c11738b5998045224829db" ], - "version": "==2.0.9" + "version": "==2.0.10" }, "ptyprocess": { "hashes": [ @@ -568,10 +568,10 @@ }, "traitlets": { "hashes": [ - "sha256:9c4bd2d267b7153df9152698efb1050a5d84982d3384a37b2c1f7723ba3e7835", - "sha256:c6cb5e6f57c5a9bdaa40fa71ce7b4af30298fbab9ece9815b5d995ab6217c7d9" + "sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44", + "sha256:d023ee369ddd2763310e4c3eae1ff649689440d4ae59d7485eb4cfbbe3e359f7" ], - "version": "==4.3.2" + "version": "==4.3.3" }, "wcwidth": { "hashes": [ diff --git a/README.md b/README.md index b95f8534..b83f36df 100644 --- a/README.md +++ b/README.md @@ -88,3 +88,4 @@ You can set environment variables in your OS, write on ```.env``` file or pass v | BOTHUB_ENGINE_AWS_ACCESS_KEY_ID | ```string``` | ```None``` | | BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY | ```string``` | ```None``` | | BOTHUB_ENGINE_AWS_REGION_NAME | ```string``` | ```None``` | +| BOTHUB_ENGINE_AWS_SEND | ```bool``` | ```False``` | diff --git a/bothub/api/v2/nlp/views.py b/bothub/api/v2/nlp/views.py index 4ce34296..1007317f 100644 --- a/bothub/api/v2/nlp/views.py +++ b/bothub/api/v2/nlp/views.py @@ -2,9 +2,12 @@ import json import requests +from django.conf import settings from django.db import models from django.utils.translation import gettext_lazy as _ from django.shortcuts import get_object_or_404 +from django.core.validators import URLValidator +from django.core.exceptions import ValidationError from rest_framework import mixins from rest_framework import exceptions @@ -417,11 +420,18 @@ class RepositoryUpdateInterpretersViewSet( def retrieve(self, request, *args, **kwargs): check_auth(request) update = self.get_object() + + validator = URLValidator() + try: + validator(str(update.bot_data)) download = requests.get(update.bot_data) bot_data = base64.b64encode(download.content) + except ValidationError: + bot_data = update.bot_data except Exception: bot_data = b"" + return Response( { "update_id": update.id, @@ -434,6 +444,9 @@ def create(self, request, *args, **kwargs): check_auth(request) id = request.data.get("id") repository = get_object_or_404(RepositoryUpdate, pk=id) - bot_data = base64.b64decode(request.data.get("bot_data")) - repository.save_training(send_bot_data_file_aws(id, bot_data)) + if settings.AWS_SEND: + bot_data = base64.b64decode(request.data.get("bot_data")) + repository.save_training(send_bot_data_file_aws(id, bot_data)) + else: + repository.save_training(request.data.get("bot_data")) return Response({}) diff --git a/bothub/common/migrations/0035_auto_20190902_1455.py b/bothub/common/migrations/0035_auto_20190902_1455.py index 826e3b8a..7013dc16 100644 --- a/bothub/common/migrations/0035_auto_20190902_1455.py +++ b/bothub/common/migrations/0035_auto_20190902_1455.py @@ -1,17 +1,18 @@ # Generated by Django 2.1.5 on 2019-09-02 14:55 - +from django.conf import settings from django.db import migrations, models from bothub.utils import send_bot_data_file_aws from bothub.common.models import RepositoryUpdate def update_repository(apps, schema_editor): - for update in RepositoryUpdate.objects.all().exclude(bot_data__exact=""): - url = send_bot_data_file_aws(update.pk, update.bot_data) - repository_update = RepositoryUpdate.objects.get(pk=update.pk) - repository_update.bot_data = url - repository_update.save(update_fields=["bot_data"]) - print("Updating bot_data repository_update {}".format(str(update.pk))) + if settings.AWS_SEND: + for update in RepositoryUpdate.objects.all().exclude(bot_data__exact=""): + repository_update = RepositoryUpdate.objects.get(pk=update.pk) + bot_data = send_bot_data_file_aws(update.pk, update.bot_data) + repository_update.bot_data = bot_data + repository_update.save(update_fields=["bot_data"]) + print("Updating bot_data repository_update {}".format(str(update.pk))) class Migration(migrations.Migration): @@ -23,6 +24,6 @@ class Migration(migrations.Migration): migrations.AlterField( model_name="repositoryupdate", name="bot_data", - field=models.URLField(blank=True, verbose_name="bot data"), + field=models.TextField(blank=True, verbose_name="bot data"), ), ] diff --git a/bothub/common/models.py b/bothub/common/models.py index 2f79fca2..b251db22 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -489,7 +489,7 @@ class Meta: use_competing_intents = models.BooleanField(default=False) use_name_entities = models.BooleanField(default=False) created_at = models.DateTimeField(_("created at"), auto_now_add=True) - bot_data = models.URLField(_("bot data"), blank=True) + bot_data = models.TextField(_("bot data"), blank=True) by = models.ForeignKey(User, models.CASCADE, blank=True, null=True) training_started_at = models.DateTimeField( _("training started at"), blank=True, null=True diff --git a/bothub/settings.py b/bothub/settings.py index 76c1347c..b2165204 100644 --- a/bothub/settings.py +++ b/bothub/settings.py @@ -44,6 +44,7 @@ BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY=(str, ""), BOTHUB_ENGINE_AWS_S3_BUCKET_NAME=(str, ""), BOTHUB_ENGINE_AWS_REGION_NAME=(str, "us-east-1"), + BOTHUB_ENGINE_AWS_SEND=(bool, False), ) # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -257,7 +258,7 @@ # AWS - +AWS_SEND = env.bool("BOTHUB_ENGINE_AWS_SEND") AWS_ACCESS_KEY_ID = env.str("BOTHUB_ENGINE_AWS_ACCESS_KEY_ID") AWS_SECRET_ACCESS_KEY = env.str("BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY") AWS_BUCKET_NAME = env.str("BOTHUB_ENGINE_AWS_S3_BUCKET_NAME")