diff --git a/Pipfile b/Pipfile index 9e440481a..b55d0cf75 100644 --- a/Pipfile +++ b/Pipfile @@ -18,6 +18,7 @@ gevent = "*" packaging = "*" django-environ = "*" boto3 = "*" +validators = "*" [dev-packages] "flake8" = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 574674e84..f0e81cc29 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "de9033e279afa3aa5dfb7a263f9e5b855543712b0cd5a20abd1cba5a45564dd6" + "sha256": "622a8920b1d5ef6db53fa5e8b3ab4b27b7c1b422d32d79624604ae0d12d23ba2" }, "pipfile-spec": 6, "requires": { @@ -18,18 +18,18 @@ "default": { "boto3": { "hashes": [ - "sha256:5ff78c697d8009b9fe9808baea60660f0a07be666fb7c65fc4c11756f568124e", - "sha256:d01314496080ac82ddff3d1b2c6ad542d89bfcb100b7a0e8aaf2d6aef99775c6" + "sha256:2efd0a9647ef8fd7bf9fbbfdeb77007050258aa39914569c689bdf9cc288cdc1", + "sha256:331b82e878f524be11c10d866a68b123b1c5e892d218a8250de10bf11cb14402" ], "index": "pypi", - "version": "==1.9.239" + "version": "==1.9.244" }, "botocore": { "hashes": [ - "sha256:14ff881779776a5b08e4bb8d7e0cd6ca86e36fc80decc536d605c406fb81c993", - "sha256:5df4a8f8bb579daed58368d6c15cdd72455bfee7dc86be243e2ef4dff5ba1177" + "sha256:7b75482156ef93e7a463f80411dc76fc868cd9fb1dedf03250cec98f8459d765", + "sha256:e43e7d19f203d572f78c18a6a16cb87bbea10bc0543e0285f7b6a35d0c825bb6" ], - "version": "==1.12.239" + "version": "==1.12.244" }, "certifi": { "hashes": [ @@ -60,6 +60,13 @@ ], "version": "==0.0.4" }, + "decorator": { + "hashes": [ + "sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de", + "sha256:f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6" + ], + "version": "==4.4.0" + }, "django": { "hashes": [ "sha256:1a41831eace203fd1939edf899e07d7abd12ce9bafc3d9a5a63a24a8d1d12bd5", @@ -110,11 +117,11 @@ }, "drf-yasg": { "hashes": [ - "sha256:68fded2ffdf46e03f33e766184b7d8f1e1a5236f94acfd0c4ba932a57b812566", - "sha256:fcef74709ead2b365410be3d12afbfd0a6e49d1efe615a15a929da7e950bb83c" + "sha256:4cfec631880ae527a91ec7cd3241aea2f82189f59e2f089119aa687761afb227", + "sha256:504cce09035cf1bace63b84d9d778b772f86bb37d8a71ed6f723346362e633b2" ], "index": "pypi", - "version": "==1.16.1" + "version": "==1.17.0" }, "gevent": { "hashes": [ @@ -199,10 +206,10 @@ }, "jinja2": { "hashes": [ - "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", - "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" + "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f", + "sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de" ], - "version": "==2.10.1" + "version": "==2.10.3" }, "jmespath": { "hashes": [ @@ -344,6 +351,13 @@ "markers": "python_version >= '3.4'", "version": "==1.24.3" }, + "validators": { + "hashes": [ + "sha256:f0ac832212e3ee2e9b10e156f19b106888cf1429c291fbc5297aae87685014ae" + ], + "index": "pypi", + "version": "==0.14.0" + }, "whitenoise": { "hashes": [ "sha256:118ab3e5f815d380171b100b05b76de2a07612f422368a201a9ffdeefb2251c1", @@ -371,10 +385,10 @@ }, "attrs": { "hashes": [ - "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", - "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + "sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2", + "sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396" ], - "version": "==19.1.0" + "version": "==19.2.0" }, "autopep8": { "hashes": [ @@ -518,11 +532,11 @@ }, "prompt-toolkit": { "hashes": [ - "sha256:11adf3389a996a6d45cc277580d0d53e8a5afd281d0c9ec71b28e6f121463780", - "sha256:2519ad1d8038fd5fc8e770362237ad0364d16a7650fb5724af6997ed5515e3c1", - "sha256:977c6583ae813a37dc1c2e1b715892461fcbdaa57f6fc62f33a528c4886c8f55" + "sha256:46642344ce457641f28fc9d1c9ca939b63dadf8df128b86f1b9860e59c73a5e4", + "sha256:e7f8af9e3d70f514373bf41aa51bc33af12a6db3f71461ea47fea985defb2c31", + "sha256:f15af68f66e664eaa559d4ac8a928111eebd5feda0c11738b5998045224829db" ], - "version": "==2.0.9" + "version": "==2.0.10" }, "ptyprocess": { "hashes": [ @@ -568,10 +582,10 @@ }, "traitlets": { "hashes": [ - "sha256:9c4bd2d267b7153df9152698efb1050a5d84982d3384a37b2c1f7723ba3e7835", - "sha256:c6cb5e6f57c5a9bdaa40fa71ce7b4af30298fbab9ece9815b5d995ab6217c7d9" + "sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44", + "sha256:d023ee369ddd2763310e4c3eae1ff649689440d4ae59d7485eb4cfbbe3e359f7" ], - "version": "==4.3.2" + "version": "==4.3.3" }, "wcwidth": { "hashes": [ diff --git a/README.md b/README.md index b95f85343..b83f36dfe 100644 --- a/README.md +++ b/README.md @@ -88,3 +88,4 @@ You can set environment variables in your OS, write on ```.env``` file or pass v | BOTHUB_ENGINE_AWS_ACCESS_KEY_ID | ```string``` | ```None``` | | BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY | ```string``` | ```None``` | | BOTHUB_ENGINE_AWS_REGION_NAME | ```string``` | ```None``` | +| BOTHUB_ENGINE_AWS_SEND | ```bool``` | ```False``` | diff --git a/bothub/api/v2/nlp/views.py b/bothub/api/v2/nlp/views.py index 4ce342969..b00903554 100644 --- a/bothub/api/v2/nlp/views.py +++ b/bothub/api/v2/nlp/views.py @@ -1,7 +1,9 @@ import base64 import json import requests +import validators +from django.conf import settings from django.db import models from django.utils.translation import gettext_lazy as _ from django.shortcuts import get_object_or_404 @@ -417,11 +419,15 @@ class RepositoryUpdateInterpretersViewSet( def retrieve(self, request, *args, **kwargs): check_auth(request) update = self.get_object() - try: - download = requests.get(update.bot_data) - bot_data = base64.b64encode(download.content) - except Exception: - bot_data = b"" + + if validators.url(str(update.bot_data).lower()): + try: + download = requests.get(update.bot_data) + bot_data = base64.b64encode(download.content) + except Exception: + bot_data = b"" + else: + bot_data = update.bot_data return Response( { "update_id": update.id, @@ -434,6 +440,9 @@ def create(self, request, *args, **kwargs): check_auth(request) id = request.data.get("id") repository = get_object_or_404(RepositoryUpdate, pk=id) - bot_data = base64.b64decode(request.data.get("bot_data")) - repository.save_training(send_bot_data_file_aws(id, bot_data)) + if settings.AWS_SEND: + bot_data = base64.b64decode(request.data.get("bot_data")) + repository.save_training(send_bot_data_file_aws(id, bot_data)) + else: + repository.save_training(request.data.get("bot_data")) return Response({}) diff --git a/bothub/common/migrations/0035_auto_20190902_1455.py b/bothub/common/migrations/0035_auto_20190902_1455.py index 826e3b8ad..7013dc164 100644 --- a/bothub/common/migrations/0035_auto_20190902_1455.py +++ b/bothub/common/migrations/0035_auto_20190902_1455.py @@ -1,17 +1,18 @@ # Generated by Django 2.1.5 on 2019-09-02 14:55 - +from django.conf import settings from django.db import migrations, models from bothub.utils import send_bot_data_file_aws from bothub.common.models import RepositoryUpdate def update_repository(apps, schema_editor): - for update in RepositoryUpdate.objects.all().exclude(bot_data__exact=""): - url = send_bot_data_file_aws(update.pk, update.bot_data) - repository_update = RepositoryUpdate.objects.get(pk=update.pk) - repository_update.bot_data = url - repository_update.save(update_fields=["bot_data"]) - print("Updating bot_data repository_update {}".format(str(update.pk))) + if settings.AWS_SEND: + for update in RepositoryUpdate.objects.all().exclude(bot_data__exact=""): + repository_update = RepositoryUpdate.objects.get(pk=update.pk) + bot_data = send_bot_data_file_aws(update.pk, update.bot_data) + repository_update.bot_data = bot_data + repository_update.save(update_fields=["bot_data"]) + print("Updating bot_data repository_update {}".format(str(update.pk))) class Migration(migrations.Migration): @@ -23,6 +24,6 @@ class Migration(migrations.Migration): migrations.AlterField( model_name="repositoryupdate", name="bot_data", - field=models.URLField(blank=True, verbose_name="bot data"), + field=models.TextField(blank=True, verbose_name="bot data"), ), ] diff --git a/bothub/common/models.py b/bothub/common/models.py index 7ac4f6639..4bbd06311 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -453,7 +453,7 @@ class Meta: use_competing_intents = models.BooleanField(default=False) use_name_entities = models.BooleanField(default=False) created_at = models.DateTimeField(_("created at"), auto_now_add=True) - bot_data = models.URLField(_("bot data"), blank=True) + bot_data = models.TextField(_("bot data"), blank=True) by = models.ForeignKey(User, models.CASCADE, blank=True, null=True) training_started_at = models.DateTimeField( _("training started at"), blank=True, null=True diff --git a/bothub/settings.py b/bothub/settings.py index dc2d90d26..0603f7a8d 100644 --- a/bothub/settings.py +++ b/bothub/settings.py @@ -44,6 +44,7 @@ BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY=(str, ""), BOTHUB_ENGINE_AWS_S3_BUCKET_NAME=(str, ""), BOTHUB_ENGINE_AWS_REGION_NAME=(str, "us-east-1"), + BOTHUB_ENGINE_AWS_SEND=(bool, False), ) # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -257,7 +258,7 @@ # AWS - +AWS_SEND = env.bool("BOTHUB_ENGINE_AWS_SEND") AWS_ACCESS_KEY_ID = env.str("BOTHUB_ENGINE_AWS_ACCESS_KEY_ID") AWS_SECRET_ACCESS_KEY = env.str("BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY") AWS_BUCKET_NAME = env.str("BOTHUB_ENGINE_AWS_S3_BUCKET_NAME")