From 9e823f124fa4881236055c42f26a4c9b1b0f0ab4 Mon Sep 17 00:00:00 2001 From: Daniel Yohan Date: Fri, 27 Sep 2019 14:28:56 -0300 Subject: [PATCH 1/7] [hotfix] Save file persistor nlp train --- README.md | 1 + bothub/api/v2/nlp/views.py | 28 ++++++++++++++----- .../migrations/0035_auto_20190902_1455.py | 19 ++++++++++--- bothub/common/models.py | 7 +++-- bothub/settings.py | 5 ++-- 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index b95f85343..b83f36dfe 100644 --- a/README.md +++ b/README.md @@ -88,3 +88,4 @@ You can set environment variables in your OS, write on ```.env``` file or pass v | BOTHUB_ENGINE_AWS_ACCESS_KEY_ID | ```string``` | ```None``` | | BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY | ```string``` | ```None``` | | BOTHUB_ENGINE_AWS_REGION_NAME | ```string``` | ```None``` | +| BOTHUB_ENGINE_AWS_SEND | ```bool``` | ```False``` | diff --git a/bothub/api/v2/nlp/views.py b/bothub/api/v2/nlp/views.py index b72a73448..a829f15c2 100644 --- a/bothub/api/v2/nlp/views.py +++ b/bothub/api/v2/nlp/views.py @@ -1,6 +1,7 @@ import base64 import json import requests +from django.conf import settings from django.db import models from django.utils.translation import gettext_lazy as _ @@ -501,11 +502,15 @@ class RepositoryUpdateInterpretersViewSet( def retrieve(self, request, *args, **kwargs): check_auth(request) update = self.get_object() - try: - download = requests.get(update.bot_data) - bot_data = base64.b64encode(download.content) - except Exception: - bot_data = b'' + + if update.is_url: + try: + download = requests.get(update.bot_data) + bot_data = base64.b64encode(download.content) + except Exception: + bot_data = b'' + else: + bot_data = update.bot_data return Response({ 'update_id': update.id, 'repository_uuid': update.repository.uuid, @@ -519,6 +524,15 @@ def create(self, request, *args, **kwargs): RepositoryUpdate, pk=id ) - bot_data = base64.b64decode(request.data.get('bot_data')) - repository.save_training(send_bot_data_file_aws(id, bot_data)) + if settings.AWS_SEND: + bot_data = base64.b64decode(request.data.get('bot_data')) + repository.save_training( + send_bot_data_file_aws( + id, + bot_data + ), + True + ) + else: + repository.save_training(request.data.get('bot_data'), False) return Response({}) diff --git a/bothub/common/migrations/0035_auto_20190902_1455.py b/bothub/common/migrations/0035_auto_20190902_1455.py index f1cef3592..722839167 100644 --- a/bothub/common/migrations/0035_auto_20190902_1455.py +++ b/bothub/common/migrations/0035_auto_20190902_1455.py @@ -1,5 +1,5 @@ # Generated by Django 2.1.5 on 2019-09-02 14:55 - +from django.conf import settings from django.db import migrations, models from bothub.utils import send_bot_data_file_aws from bothub.common.models import RepositoryUpdate @@ -7,12 +7,18 @@ def update_repository(apps, schema_editor): for update in RepositoryUpdate.objects.all().exclude(bot_data__exact=''): - url = send_bot_data_file_aws(update.pk, update.bot_data) repository_update = RepositoryUpdate.objects.get(pk=update.pk) - repository_update.bot_data = url + if settings.AWS_SEND: + bot_data = send_bot_data_file_aws(update.pk, update.bot_data) + repository_update.is_url = True + else: + bot_data = update.bot_data + repository_update.is_url = False + repository_update.bot_data = bot_data repository_update.save( update_fields=[ 'bot_data', + 'is_url', ]) print('Updating bot_data repository_update {}'.format(str(update.pk))) @@ -24,10 +30,15 @@ class Migration(migrations.Migration): ] operations = [ + migrations.AddField( + model_name='repositoryupdate', + name='is_url', + field=models.BooleanField(default=True), + ), migrations.RunPython(update_repository), migrations.AlterField( model_name='repositoryupdate', name='bot_data', - field=models.URLField(blank=True, verbose_name='bot data'), + field=models.TextField(blank=True, verbose_name='bot data'), ), ] diff --git a/bothub/common/models.py b/bothub/common/models.py index 0c593b01e..10218d7ca 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -484,9 +484,10 @@ class Meta: created_at = models.DateTimeField( _('created at'), auto_now_add=True) - bot_data = models.URLField( + bot_data = models.TextField( _('bot data'), blank=True) + is_url = models.BooleanField(default=True) by = models.ForeignKey( User, models.CASCADE, @@ -649,18 +650,20 @@ def start_training(self, by): 'use_name_entities', ]) - def save_training(self, bot_data): + def save_training(self, bot_data, is_url): if self.trained_at: raise RepositoryUpdateAlreadyTrained() self.trained_at = timezone.now() self.bot_data = bot_data + self.is_url = is_url self.repository.total_updates += 1 self.repository.save() self.save( update_fields=[ 'trained_at', 'bot_data', + 'is_url', ]) def get_bot_data(self): diff --git a/bothub/settings.py b/bothub/settings.py index 0f908f4bc..6867cf227 100644 --- a/bothub/settings.py +++ b/bothub/settings.py @@ -40,7 +40,8 @@ BOTHUB_ENGINE_AWS_ACCESS_KEY_ID=(str, ''), BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY=(str, ''), BOTHUB_ENGINE_AWS_S3_BUCKET_NAME=(str, ''), - BOTHUB_ENGINE_AWS_REGION_NAME=(str, 'us-east-1') + BOTHUB_ENGINE_AWS_REGION_NAME=(str, 'us-east-1'), + BOTHUB_ENGINE_AWS_SEND=(bool, False) ) # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -275,7 +276,7 @@ # AWS - +AWS_SEND = env.bool('BOTHUB_ENGINE_AWS_SEND') AWS_ACCESS_KEY_ID = env.str('BOTHUB_ENGINE_AWS_ACCESS_KEY_ID') AWS_SECRET_ACCESS_KEY = env.str('BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY') AWS_BUCKET_NAME = env.str('BOTHUB_ENGINE_AWS_S3_BUCKET_NAME') From 0e51760f61d48335815493f780aed8f35610b5a3 Mon Sep 17 00:00:00 2001 From: Daniel Yohan Date: Fri, 27 Sep 2019 16:07:15 -0300 Subject: [PATCH 2/7] [hotfix] Persistor Save --- bothub/api/v2/nlp/views.py | 18 +++++++++--- .../migrations/0035_auto_20190902_1455.py | 28 ++++++------------- bothub/common/models.py | 5 +--- 3 files changed, 24 insertions(+), 27 deletions(-) diff --git a/bothub/api/v2/nlp/views.py b/bothub/api/v2/nlp/views.py index a829f15c2..20e3e7893 100644 --- a/bothub/api/v2/nlp/views.py +++ b/bothub/api/v2/nlp/views.py @@ -1,5 +1,7 @@ import base64 import json +import re + import requests from django.conf import settings @@ -503,7 +505,16 @@ def retrieve(self, request, *args, **kwargs): check_auth(request) update = self.get_object() - if update.is_url: + regex = re.compile( + r'^(?:http|ftp)s?://' # http:// or https:// + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|' + r'[A-Z0-9-]{2,}\.?)|' + r'localhost|' + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' + r'(?::\d+)?' + r'(?:/?|[/?]\S+)$', re.IGNORECASE) + + if re.match(regex, update.bot_data) is not None: try: download = requests.get(update.bot_data) bot_data = base64.b64encode(download.content) @@ -530,9 +541,8 @@ def create(self, request, *args, **kwargs): send_bot_data_file_aws( id, bot_data - ), - True + ) ) else: - repository.save_training(request.data.get('bot_data'), False) + repository.save_training(request.data.get('bot_data')) return Response({}) diff --git a/bothub/common/migrations/0035_auto_20190902_1455.py b/bothub/common/migrations/0035_auto_20190902_1455.py index 722839167..f033d0ba5 100644 --- a/bothub/common/migrations/0035_auto_20190902_1455.py +++ b/bothub/common/migrations/0035_auto_20190902_1455.py @@ -6,21 +6,16 @@ def update_repository(apps, schema_editor): - for update in RepositoryUpdate.objects.all().exclude(bot_data__exact=''): - repository_update = RepositoryUpdate.objects.get(pk=update.pk) - if settings.AWS_SEND: + if settings.AWS_SEND: + for update in RepositoryUpdate.objects.all().exclude(bot_data__exact=''): + repository_update = RepositoryUpdate.objects.get(pk=update.pk) bot_data = send_bot_data_file_aws(update.pk, update.bot_data) - repository_update.is_url = True - else: - bot_data = update.bot_data - repository_update.is_url = False - repository_update.bot_data = bot_data - repository_update.save( - update_fields=[ - 'bot_data', - 'is_url', - ]) - print('Updating bot_data repository_update {}'.format(str(update.pk))) + repository_update.bot_data = bot_data + repository_update.save( + update_fields=[ + 'bot_data', + ]) + print('Updating bot_data repository_update {}'.format(str(update.pk))) class Migration(migrations.Migration): @@ -30,11 +25,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AddField( - model_name='repositoryupdate', - name='is_url', - field=models.BooleanField(default=True), - ), migrations.RunPython(update_repository), migrations.AlterField( model_name='repositoryupdate', diff --git a/bothub/common/models.py b/bothub/common/models.py index 10218d7ca..291a8a5a6 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -487,7 +487,6 @@ class Meta: bot_data = models.TextField( _('bot data'), blank=True) - is_url = models.BooleanField(default=True) by = models.ForeignKey( User, models.CASCADE, @@ -650,20 +649,18 @@ def start_training(self, by): 'use_name_entities', ]) - def save_training(self, bot_data, is_url): + def save_training(self, bot_data): if self.trained_at: raise RepositoryUpdateAlreadyTrained() self.trained_at = timezone.now() self.bot_data = bot_data - self.is_url = is_url self.repository.total_updates += 1 self.repository.save() self.save( update_fields=[ 'trained_at', 'bot_data', - 'is_url', ]) def get_bot_data(self): From 8d6340d0fe891513b72b6d8a8df8eae0035ca8ed Mon Sep 17 00:00:00 2001 From: Daniel Yohan Date: Tue, 1 Oct 2019 09:59:15 -0300 Subject: [PATCH 3/7] Updated imports --- bothub/api/v2/nlp/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bothub/api/v2/nlp/views.py b/bothub/api/v2/nlp/views.py index 20e3e7893..daab02903 100644 --- a/bothub/api/v2/nlp/views.py +++ b/bothub/api/v2/nlp/views.py @@ -1,8 +1,8 @@ import base64 import json import re - import requests + from django.conf import settings from django.db import models From f2d7cb4f83694fa9a360a1381b7698aa73e2b257 Mon Sep 17 00:00:00 2001 From: Daniel Yohan Date: Tue, 1 Oct 2019 10:00:31 -0300 Subject: [PATCH 4/7] Updated imports --- bothub/api/v2/nlp/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bothub/api/v2/nlp/views.py b/bothub/api/v2/nlp/views.py index daab02903..5a947ee50 100644 --- a/bothub/api/v2/nlp/views.py +++ b/bothub/api/v2/nlp/views.py @@ -4,7 +4,6 @@ import requests from django.conf import settings - from django.db import models from django.utils.translation import gettext_lazy as _ from django.shortcuts import get_object_or_404 From 77e7d1bd2ae0e4501f39f5ea12e1eadabf17ffbd Mon Sep 17 00:00:00 2001 From: Daniel Yohan Date: Tue, 1 Oct 2019 16:06:06 -0300 Subject: [PATCH 5/7] Updated PEP8 --- bothub/api/v2/nlp/views.py | 48 +++++++++---------- .../migrations/0035_auto_20190902_1455.py | 15 +++--- bothub/common/models.py | 14 ++---- bothub/settings.py | 20 ++++---- 4 files changed, 41 insertions(+), 56 deletions(-) diff --git a/bothub/api/v2/nlp/views.py b/bothub/api/v2/nlp/views.py index 919a992be..2a2bd7685 100644 --- a/bothub/api/v2/nlp/views.py +++ b/bothub/api/v2/nlp/views.py @@ -421,43 +421,39 @@ def retrieve(self, request, *args, **kwargs): update = self.get_object() regex = re.compile( - r'^(?:http|ftp)s?://' # http:// or https:// - r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|' - r'[A-Z0-9-]{2,}\.?)|' - r'localhost|' - r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' - r'(?::\d+)?' - r'(?:/?|[/?]\S+)$', re.IGNORECASE) + r"^(?:http|ftp)s?://" # http:// or https:// + r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|" + r"[A-Z0-9-]{2,}\.?)|" + r"localhost|" + r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})" + r"(?::\d+)?" + r"(?:/?|[/?]\S+)$", + re.IGNORECASE, + ) if re.match(regex, update.bot_data) is not None: try: download = requests.get(update.bot_data) bot_data = base64.b64encode(download.content) except Exception: - bot_data = b'' + bot_data = b"" else: bot_data = update.bot_data - return Response({ - 'update_id': update.id, - 'repository_uuid': update.repository.uuid, - 'bot_data': str(bot_data) - }) + return Response( + { + "update_id": update.id, + "repository_uuid": update.repository.uuid, + "bot_data": str(bot_data), + } + ) def create(self, request, *args, **kwargs): check_auth(request) - id = request.data.get('id') - repository = get_object_or_404( - RepositoryUpdate, - pk=id - ) + id = request.data.get("id") + repository = get_object_or_404(RepositoryUpdate, pk=id) if settings.AWS_SEND: - bot_data = base64.b64decode(request.data.get('bot_data')) - repository.save_training( - send_bot_data_file_aws( - id, - bot_data - ) - ) + bot_data = base64.b64decode(request.data.get("bot_data")) + repository.save_training(send_bot_data_file_aws(id, bot_data)) else: - repository.save_training(request.data.get('bot_data')) + repository.save_training(request.data.get("bot_data")) return Response({}) diff --git a/bothub/common/migrations/0035_auto_20190902_1455.py b/bothub/common/migrations/0035_auto_20190902_1455.py index 2afd01c58..7013dc164 100644 --- a/bothub/common/migrations/0035_auto_20190902_1455.py +++ b/bothub/common/migrations/0035_auto_20190902_1455.py @@ -7,15 +7,12 @@ def update_repository(apps, schema_editor): if settings.AWS_SEND: - for update in RepositoryUpdate.objects.all().exclude(bot_data__exact=''): + for update in RepositoryUpdate.objects.all().exclude(bot_data__exact=""): repository_update = RepositoryUpdate.objects.get(pk=update.pk) bot_data = send_bot_data_file_aws(update.pk, update.bot_data) repository_update.bot_data = bot_data - repository_update.save( - update_fields=[ - 'bot_data', - ]) - print('Updating bot_data repository_update {}'.format(str(update.pk))) + repository_update.save(update_fields=["bot_data"]) + print("Updating bot_data repository_update {}".format(str(update.pk))) class Migration(migrations.Migration): @@ -25,8 +22,8 @@ class Migration(migrations.Migration): operations = [ migrations.RunPython(update_repository), migrations.AlterField( - model_name='repositoryupdate', - name='bot_data', - field=models.TextField(blank=True, verbose_name='bot data'), + model_name="repositoryupdate", + name="bot_data", + field=models.TextField(blank=True, verbose_name="bot data"), ), ] diff --git a/bothub/common/models.py b/bothub/common/models.py index 1ec118aea..4bbd06311 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -452,17 +452,9 @@ class Meta: ) use_competing_intents = models.BooleanField(default=False) use_name_entities = models.BooleanField(default=False) - created_at = models.DateTimeField( - _('created at'), - auto_now_add=True) - bot_data = models.TextField( - _('bot data'), - blank=True) - by = models.ForeignKey( - User, - models.CASCADE, - blank=True, - null=True) + created_at = models.DateTimeField(_("created at"), auto_now_add=True) + bot_data = models.TextField(_("bot data"), blank=True) + by = models.ForeignKey(User, models.CASCADE, blank=True, null=True) training_started_at = models.DateTimeField( _("training started at"), blank=True, null=True ) diff --git a/bothub/settings.py b/bothub/settings.py index b89ff6a3c..0603f7a8d 100644 --- a/bothub/settings.py +++ b/bothub/settings.py @@ -40,11 +40,11 @@ CSRF_COOKIE_SECURE=(bool, False), SUPPORTED_LANGUAGES=(cast_supported_languages, "en|pt"), CHECK_ACCESSIBLE_API_URL=(str, None), - BOTHUB_ENGINE_AWS_ACCESS_KEY_ID=(str, ''), - BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY=(str, ''), - BOTHUB_ENGINE_AWS_S3_BUCKET_NAME=(str, ''), - BOTHUB_ENGINE_AWS_REGION_NAME=(str, 'us-east-1'), - BOTHUB_ENGINE_AWS_SEND=(bool, False) + BOTHUB_ENGINE_AWS_ACCESS_KEY_ID=(str, ""), + BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY=(str, ""), + BOTHUB_ENGINE_AWS_S3_BUCKET_NAME=(str, ""), + BOTHUB_ENGINE_AWS_REGION_NAME=(str, "us-east-1"), + BOTHUB_ENGINE_AWS_SEND=(bool, False), ) # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -258,8 +258,8 @@ # AWS -AWS_SEND = env.bool('BOTHUB_ENGINE_AWS_SEND') -AWS_ACCESS_KEY_ID = env.str('BOTHUB_ENGINE_AWS_ACCESS_KEY_ID') -AWS_SECRET_ACCESS_KEY = env.str('BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY') -AWS_BUCKET_NAME = env.str('BOTHUB_ENGINE_AWS_S3_BUCKET_NAME') -AWS_REGION_NAME = env.str('BOTHUB_ENGINE_AWS_REGION_NAME') +AWS_SEND = env.bool("BOTHUB_ENGINE_AWS_SEND") +AWS_ACCESS_KEY_ID = env.str("BOTHUB_ENGINE_AWS_ACCESS_KEY_ID") +AWS_SECRET_ACCESS_KEY = env.str("BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY") +AWS_BUCKET_NAME = env.str("BOTHUB_ENGINE_AWS_S3_BUCKET_NAME") +AWS_REGION_NAME = env.str("BOTHUB_ENGINE_AWS_REGION_NAME") From 1a5290d98e17314b12d7cca97fcb564265b7525a Mon Sep 17 00:00:00 2001 From: Daniel Yohan Date: Tue, 8 Oct 2019 08:56:47 -0300 Subject: [PATCH 6/7] Update Regex check url AWS --- Pipfile | 1 + Pipfile.lock | 60 +++++++++++++++++++++++--------------- bothub/api/v2/nlp/views.py | 15 ++-------- 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/Pipfile b/Pipfile index 9e440481a..b55d0cf75 100644 --- a/Pipfile +++ b/Pipfile @@ -18,6 +18,7 @@ gevent = "*" packaging = "*" django-environ = "*" boto3 = "*" +validators = "*" [dev-packages] "flake8" = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 574674e84..f0e81cc29 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "de9033e279afa3aa5dfb7a263f9e5b855543712b0cd5a20abd1cba5a45564dd6" + "sha256": "622a8920b1d5ef6db53fa5e8b3ab4b27b7c1b422d32d79624604ae0d12d23ba2" }, "pipfile-spec": 6, "requires": { @@ -18,18 +18,18 @@ "default": { "boto3": { "hashes": [ - "sha256:5ff78c697d8009b9fe9808baea60660f0a07be666fb7c65fc4c11756f568124e", - "sha256:d01314496080ac82ddff3d1b2c6ad542d89bfcb100b7a0e8aaf2d6aef99775c6" + "sha256:2efd0a9647ef8fd7bf9fbbfdeb77007050258aa39914569c689bdf9cc288cdc1", + "sha256:331b82e878f524be11c10d866a68b123b1c5e892d218a8250de10bf11cb14402" ], "index": "pypi", - "version": "==1.9.239" + "version": "==1.9.244" }, "botocore": { "hashes": [ - "sha256:14ff881779776a5b08e4bb8d7e0cd6ca86e36fc80decc536d605c406fb81c993", - "sha256:5df4a8f8bb579daed58368d6c15cdd72455bfee7dc86be243e2ef4dff5ba1177" + "sha256:7b75482156ef93e7a463f80411dc76fc868cd9fb1dedf03250cec98f8459d765", + "sha256:e43e7d19f203d572f78c18a6a16cb87bbea10bc0543e0285f7b6a35d0c825bb6" ], - "version": "==1.12.239" + "version": "==1.12.244" }, "certifi": { "hashes": [ @@ -60,6 +60,13 @@ ], "version": "==0.0.4" }, + "decorator": { + "hashes": [ + "sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de", + "sha256:f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6" + ], + "version": "==4.4.0" + }, "django": { "hashes": [ "sha256:1a41831eace203fd1939edf899e07d7abd12ce9bafc3d9a5a63a24a8d1d12bd5", @@ -110,11 +117,11 @@ }, "drf-yasg": { "hashes": [ - "sha256:68fded2ffdf46e03f33e766184b7d8f1e1a5236f94acfd0c4ba932a57b812566", - "sha256:fcef74709ead2b365410be3d12afbfd0a6e49d1efe615a15a929da7e950bb83c" + "sha256:4cfec631880ae527a91ec7cd3241aea2f82189f59e2f089119aa687761afb227", + "sha256:504cce09035cf1bace63b84d9d778b772f86bb37d8a71ed6f723346362e633b2" ], "index": "pypi", - "version": "==1.16.1" + "version": "==1.17.0" }, "gevent": { "hashes": [ @@ -199,10 +206,10 @@ }, "jinja2": { "hashes": [ - "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", - "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" + "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f", + "sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de" ], - "version": "==2.10.1" + "version": "==2.10.3" }, "jmespath": { "hashes": [ @@ -344,6 +351,13 @@ "markers": "python_version >= '3.4'", "version": "==1.24.3" }, + "validators": { + "hashes": [ + "sha256:f0ac832212e3ee2e9b10e156f19b106888cf1429c291fbc5297aae87685014ae" + ], + "index": "pypi", + "version": "==0.14.0" + }, "whitenoise": { "hashes": [ "sha256:118ab3e5f815d380171b100b05b76de2a07612f422368a201a9ffdeefb2251c1", @@ -371,10 +385,10 @@ }, "attrs": { "hashes": [ - "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", - "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + "sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2", + "sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396" ], - "version": "==19.1.0" + "version": "==19.2.0" }, "autopep8": { "hashes": [ @@ -518,11 +532,11 @@ }, "prompt-toolkit": { "hashes": [ - "sha256:11adf3389a996a6d45cc277580d0d53e8a5afd281d0c9ec71b28e6f121463780", - "sha256:2519ad1d8038fd5fc8e770362237ad0364d16a7650fb5724af6997ed5515e3c1", - "sha256:977c6583ae813a37dc1c2e1b715892461fcbdaa57f6fc62f33a528c4886c8f55" + "sha256:46642344ce457641f28fc9d1c9ca939b63dadf8df128b86f1b9860e59c73a5e4", + "sha256:e7f8af9e3d70f514373bf41aa51bc33af12a6db3f71461ea47fea985defb2c31", + "sha256:f15af68f66e664eaa559d4ac8a928111eebd5feda0c11738b5998045224829db" ], - "version": "==2.0.9" + "version": "==2.0.10" }, "ptyprocess": { "hashes": [ @@ -568,10 +582,10 @@ }, "traitlets": { "hashes": [ - "sha256:9c4bd2d267b7153df9152698efb1050a5d84982d3384a37b2c1f7723ba3e7835", - "sha256:c6cb5e6f57c5a9bdaa40fa71ce7b4af30298fbab9ece9815b5d995ab6217c7d9" + "sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44", + "sha256:d023ee369ddd2763310e4c3eae1ff649689440d4ae59d7485eb4cfbbe3e359f7" ], - "version": "==4.3.2" + "version": "==4.3.3" }, "wcwidth": { "hashes": [ diff --git a/bothub/api/v2/nlp/views.py b/bothub/api/v2/nlp/views.py index 2a2bd7685..b00903554 100644 --- a/bothub/api/v2/nlp/views.py +++ b/bothub/api/v2/nlp/views.py @@ -1,7 +1,7 @@ import base64 import json -import re import requests +import validators from django.conf import settings from django.db import models @@ -420,18 +420,7 @@ def retrieve(self, request, *args, **kwargs): check_auth(request) update = self.get_object() - regex = re.compile( - r"^(?:http|ftp)s?://" # http:// or https:// - r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|" - r"[A-Z0-9-]{2,}\.?)|" - r"localhost|" - r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})" - r"(?::\d+)?" - r"(?:/?|[/?]\S+)$", - re.IGNORECASE, - ) - - if re.match(regex, update.bot_data) is not None: + if validators.url(str(update.bot_data).lower()): try: download = requests.get(update.bot_data) bot_data = base64.b64encode(download.content) From 943ff68e2cca6b77ad732e4f6282c9c3c3f6339e Mon Sep 17 00:00:00 2001 From: Daniel Yohan Date: Tue, 8 Oct 2019 09:09:13 -0300 Subject: [PATCH 7/7] Updated Check URL --- Pipfile | 1 - Pipfile.lock | 16 +--------------- bothub/api/v2/nlp/views.py | 13 +++++++++++-- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/Pipfile b/Pipfile index b55d0cf75..9e440481a 100644 --- a/Pipfile +++ b/Pipfile @@ -18,7 +18,6 @@ gevent = "*" packaging = "*" django-environ = "*" boto3 = "*" -validators = "*" [dev-packages] "flake8" = "*" diff --git a/Pipfile.lock b/Pipfile.lock index f0e81cc29..4bf45f763 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "622a8920b1d5ef6db53fa5e8b3ab4b27b7c1b422d32d79624604ae0d12d23ba2" + "sha256": "de9033e279afa3aa5dfb7a263f9e5b855543712b0cd5a20abd1cba5a45564dd6" }, "pipfile-spec": 6, "requires": { @@ -60,13 +60,6 @@ ], "version": "==0.0.4" }, - "decorator": { - "hashes": [ - "sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de", - "sha256:f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6" - ], - "version": "==4.4.0" - }, "django": { "hashes": [ "sha256:1a41831eace203fd1939edf899e07d7abd12ce9bafc3d9a5a63a24a8d1d12bd5", @@ -351,13 +344,6 @@ "markers": "python_version >= '3.4'", "version": "==1.24.3" }, - "validators": { - "hashes": [ - "sha256:f0ac832212e3ee2e9b10e156f19b106888cf1429c291fbc5297aae87685014ae" - ], - "index": "pypi", - "version": "==0.14.0" - }, "whitenoise": { "hashes": [ "sha256:118ab3e5f815d380171b100b05b76de2a07612f422368a201a9ffdeefb2251c1", diff --git a/bothub/api/v2/nlp/views.py b/bothub/api/v2/nlp/views.py index b00903554..fc4fa9ce3 100644 --- a/bothub/api/v2/nlp/views.py +++ b/bothub/api/v2/nlp/views.py @@ -1,12 +1,13 @@ import base64 import json import requests -import validators from django.conf import settings from django.db import models from django.utils.translation import gettext_lazy as _ from django.shortcuts import get_object_or_404 +from django.core.validators import URLValidator +from django.core.exceptions import ValidationError from rest_framework import mixins from rest_framework import exceptions @@ -420,7 +421,15 @@ def retrieve(self, request, *args, **kwargs): check_auth(request) update = self.get_object() - if validators.url(str(update.bot_data).lower()): + validator = URLValidator() + + try: + validator(str(update.bot_data)) + url_valid = True + except ValidationError: + url_valid = False + + if url_valid: try: download = requests.get(update.bot_data) bot_data = base64.b64encode(download.content)