From 13eb97f0a08014f6cbfbab0f7062afebce2a63c0 Mon Sep 17 00:00:00 2001 From: Daniel Yohan Date: Fri, 30 Aug 2019 16:39:45 -0300 Subject: [PATCH 01/10] [fix] Conflicts routers repository --- bothub/api/v2/routers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bothub/api/v2/routers.py b/bothub/api/v2/routers.py index 08923d2c..5f45520b 100644 --- a/bothub/api/v2/routers.py +++ b/bothub/api/v2/routers.py @@ -99,7 +99,7 @@ def get_lookup_regex(self, viewset, lookup_prefix=''): router = Router() -router.register('repository', RepositoryViewSet) +router.register('repository/repository-info', RepositoryViewSet) router.register('repository/repository-votes', RepositoryVotesViewSet) router.register('repository/repositories', RepositoriesViewSet) router.register( From 74aac6277db1957a42d2baa8ee20d12399e019d7 Mon Sep 17 00:00:00 2001 From: Daniel Yohan Date: Fri, 30 Aug 2019 16:40:01 -0300 Subject: [PATCH 02/10] added router upload examples with json --- bothub/api/v2/repository/views.py | 48 +++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/bothub/api/v2/repository/views.py b/bothub/api/v2/repository/views.py index 84147c09..c55c15cc 100644 --- a/bothub/api/v2/repository/views.py +++ b/bothub/api/v2/repository/views.py @@ -1,3 +1,4 @@ +import json from django.utils.decorators import method_decorator from django_filters.rest_framework import DjangoFilterBackend from django.shortcuts import get_object_or_404 @@ -7,10 +8,12 @@ from drf_yasg.utils import swagger_auto_schema from rest_framework.decorators import action from rest_framework.exceptions import ValidationError +from rest_framework.exceptions import UnsupportedMediaType from rest_framework.exceptions import PermissionDenied from rest_framework.response import Response from rest_framework.viewsets import GenericViewSet from rest_framework import mixins +from rest_framework import parsers from rest_framework import status from rest_framework import permissions from rest_framework.permissions import IsAuthenticatedOrReadOnly @@ -475,6 +478,51 @@ def create(self, request, *args, **kwargs): self.permission_classes = [permissions.IsAuthenticated] return super().create(request, *args, **kwargs) + @action( + detail=True, + methods=['POST'], + url_name='repository-upload-examples', + parser_classes=[parsers.MultiPartParser]) + def upload_examples(self, request, **kwargs): + try: + repository = get_object_or_404( + Repository, + pk=request.data.get('repository') + ) + except DjangoValidationError: + raise PermissionDenied() + + user_authorization = repository.get_user_authorization(request.user) + if not user_authorization.can_write: + raise PermissionDenied() + + f = request.FILES.get('file') + try: + json_data = json.loads(f.read()) + except json.decoder.JSONDecodeError: + raise UnsupportedMediaType('json') + + count_added = 0 + not_added = [] + + for data in json_data: + response_data = data + response_data['repository'] = request.data.get('repository') + serializer = RepositoryExampleSerializer( + data=response_data, + context={'request': request} + ) + if serializer.is_valid(): + serializer.save() + count_added += 1 + else: + not_added.append(data) + + return Response({ + 'added': count_added, + 'not_added': not_added + }) + def perform_destroy(self, obj): if obj.deleted_in: raise APIException(_('Example already deleted')) From 7694797b5a3e7768835b69154a66c866db730bcc Mon Sep 17 00:00:00 2001 From: Daniel Yohan Date: Fri, 30 Aug 2019 17:24:13 -0300 Subject: [PATCH 03/10] Added tests upload file examples --- bothub/api/v2/repository/views.py | 2 +- bothub/api/v2/tests/test_repository.py | 83 ++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/bothub/api/v2/repository/views.py b/bothub/api/v2/repository/views.py index c55c15cc..0e933709 100644 --- a/bothub/api/v2/repository/views.py +++ b/bothub/api/v2/repository/views.py @@ -498,7 +498,7 @@ def upload_examples(self, request, **kwargs): f = request.FILES.get('file') try: - json_data = json.loads(f.read()) + json_data = json.loads(f.read().decode()) except json.decoder.JSONDecodeError: raise UnsupportedMediaType('json') diff --git a/bothub/api/v2/tests/test_repository.py b/bothub/api/v2/tests/test_repository.py index 365c0a1f..d5abe902 100644 --- a/bothub/api/v2/tests/test_repository.py +++ b/bothub/api/v2/tests/test_repository.py @@ -1,6 +1,7 @@ import json import uuid +from django.core.files.uploadedfile import SimpleUploadedFile from django.test import TestCase from django.test import RequestFactory from django.test.client import MULTIPART_CONTENT @@ -1453,6 +1454,88 @@ def test_entity_has_valid_label(self): label) +class RepositoryExampleUploadTestCase(TestCase): + def setUp(self): + self.factory = RequestFactory() + + self.owner, self.owner_token = create_user_and_token('owner') + self.user, self.user_token = create_user_and_token() + + self.repository = Repository.objects.create( + owner=self.owner, + name='Testing', + slug='test', + language=languages.LANGUAGE_EN) + + def request(self, token): + authorization_header = { + 'HTTP_AUTHORIZATION': 'Token {}'.format(token.key), + } + examples = b'''[ + { + "text": "yes", + "language": "en", + "entities": [{ + "label": "yes", + "entity": "_yes", + "start": 0, + "end": 3 + }], + "intent": "greet" + }, + { + "text": "alright", + "language": "en", + "entities": [{ + "label": "yes", + "entity": "_yes", + "start": 0, + "end": 3 + }], + "intent": "greet" + } + ]''' + + uploaded_file = SimpleUploadedFile( + 'examples.json', + examples, + 'multipart/form-data' + ) + + request = self.factory.post( + '/v2/repository/example/upload_examples/', + { + 'file': uploaded_file, + 'repository': str(self.repository.uuid) + }, format='multipart', + **authorization_header) + response = RepositoryExampleViewSet.as_view( + {'post': 'upload_examples'})(request) + response.render() + content_data = json.loads(response.content) + return (response, content_data,) + + def test_okay(self): + response, content_data = self.request(self.owner_token) + self.assertEqual( + content_data.get('added'), + 2 + ) + self.assertEqual( + len(content_data.get('not_added')), + 0 + ) + self.assertEqual( + response.status_code, + status.HTTP_200_OK) + + def test_permission_denied(self): + response, content_data = self.request(self.user_token) + self.assertEqual( + response.status_code, + status.HTTP_403_FORBIDDEN) + + class RepositoryExampleDestroyTestCase(TestCase): def setUp(self): self.factory = RequestFactory() From 59b0b5a33320b85fdeedbd7ee920ca15c151d856 Mon Sep 17 00:00:00 2001 From: Daniel Yohan Date: Mon, 2 Sep 2019 08:23:00 -0300 Subject: [PATCH 04/10] [fix] Repository Upload swagger doc --- bothub/api/v2/repository/serializers.py | 4 ++++ bothub/api/v2/repository/views.py | 4 +++- bothub/api/v2/tests/test_repository.py | 18 +++++++++--------- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/bothub/api/v2/repository/serializers.py b/bothub/api/v2/repository/serializers.py index 7d019182..c3f39f30 100644 --- a/bothub/api/v2/repository/serializers.py +++ b/bothub/api/v2/repository/serializers.py @@ -595,3 +595,7 @@ class Meta: source='by', slug_field='nickname', read_only=True) + + +class RepositoryUpload(serializers.Serializer): + pass diff --git a/bothub/api/v2/repository/views.py b/bothub/api/v2/repository/views.py index 0e933709..0d1bb80a 100644 --- a/bothub/api/v2/repository/views.py +++ b/bothub/api/v2/repository/views.py @@ -44,6 +44,7 @@ from .serializers import AnalyzeTextSerializer from .serializers import EvaluateSerializer from .serializers import RepositoryUpdateSerializer +from .serializers import RepositoryUpload from .permissions import RepositoryPermission from .permissions import RepositoryAdminManagerAuthorization from .permissions import RepositoryExamplePermission @@ -482,7 +483,8 @@ def create(self, request, *args, **kwargs): detail=True, methods=['POST'], url_name='repository-upload-examples', - parser_classes=[parsers.MultiPartParser]) + parser_classes=[parsers.MultiPartParser], + serializer_class=RepositoryUpload) def upload_examples(self, request, **kwargs): try: repository = get_object_or_404( diff --git a/bothub/api/v2/tests/test_repository.py b/bothub/api/v2/tests/test_repository.py index d5abe902..c7b57601 100644 --- a/bothub/api/v2/tests/test_repository.py +++ b/bothub/api/v2/tests/test_repository.py @@ -127,7 +127,7 @@ def request(self, data, token=None): } if token else {} request = self.factory.post( - '/v2/repository/', + '/v2/repository/repository-info/', data, **authorization_header) @@ -191,7 +191,7 @@ def request(self, repository, token=None): } if token else {} request = self.factory.get( - '/v2/repository/{}/'.format(repository.uuid), + '/v2/repository/repository-info/{}/'.format(repository.uuid), **authorization_header) response = RepositoryViewSet.as_view({'get': 'retrieve'})( @@ -236,7 +236,7 @@ def request(self, repository, data={}, token=None): } if token else {} request = self.factory.patch( - '/v2/repository/{}/'.format(repository.uuid), + '/v2/repository/repository-info/{}/'.format(repository.uuid), self.factory._encode_data(data, MULTIPART_CONTENT), MULTIPART_CONTENT, **authorization_header) @@ -295,7 +295,7 @@ def request(self, repository, token=None): } if token else {} request = self.factory.get( - '/v2/repository/{}/'.format(repository.uuid), + '/v2/repository/repository-info/{}/'.format(repository.uuid), **authorization_header) response = RepositoryViewSet.as_view({'get': 'retrieve'})( @@ -344,7 +344,7 @@ def request(self, repository, token=None): } if token else {} request = self.factory.get( - '/v2/repository/{}/'.format(repository.uuid), + '/v2/repository/repository-info/{}/'.format(repository.uuid), **authorization_header) response = RepositoryViewSet.as_view({'get': 'retrieve'})( @@ -2036,7 +2036,7 @@ def request(self, repository, token): 'HTTP_AUTHORIZATION': 'Token {}'.format(token.key), } request = self.factory.get( - '/v2/repository/{}/'.format( + '/v2/repository/repository-info/{}/'.format( str(repository.uuid)), **authorization_header) response = RepositoryViewSet.as_view( @@ -2086,7 +2086,7 @@ def test_languages_status(self): 'HTTP_AUTHORIZATION': 'Token {}'.format(self.user_token.key), } request = self.factory.get( - '/v2/repository/{}/languagesstatus/'.format( + '/v2/repository/repository-info/{}/languagesstatus/'.format( self.repository.uuid), **authorization_header) response = RepositoryViewSet.as_view( @@ -2170,7 +2170,7 @@ def request(self, repository, token): 'HTTP_AUTHORIZATION': 'Token {}'.format(token.key), } request = self.factory.get( - '/v2/repository/{}/train/'.format( + '/v2/repository/repository-info/{}/train/'.format( str(repository.uuid)), **authorization_header) response = RepositoryViewSet.as_view({'get': 'train'})(request) @@ -2209,7 +2209,7 @@ def request(self, repository, token, data): 'HTTP_AUTHORIZATION': 'Token {}'.format(token.key), } request = self.factory.post( - '/v2/repository/{}/analyze/'.format( + '/v2/repository/repository-info/{}/analyze/'.format( str(repository.uuid)), data, **authorization_header) From add0f6ca22a5c760cbdc016f1e9ac9fbb2a6b9cf Mon Sep 17 00:00:00 2001 From: Daniel Yohan Date: Tue, 17 Sep 2019 11:37:34 -0300 Subject: [PATCH 05/10] [fix] Request NLP --- bothub/api/v1/views.py | 6 +++--- bothub/api/v2/repository/views.py | 6 +++--- bothub/common/models.py | 34 ++++++++++++++----------------- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/bothub/api/v1/views.py b/bothub/api/v1/views.py index 8bb5a17e..04c38012 100644 --- a/bothub/api/v1/views.py +++ b/bothub/api/v1/views.py @@ -562,7 +562,7 @@ def train(self, request, **kwargs): user_authorization = repository.get_user_authorization(request.user) if not user_authorization.can_write: raise PermissionDenied() - request = Repository.request_nlp_train( # pragma: no cover + request = repository.request_nlp_train( # pragma: no cover user_authorization) if request.status_code != status.HTTP_200_OK: # pragma: no cover raise APIException( # pragma: no cover @@ -587,7 +587,7 @@ def analyze(self, request, **kwargs): serializer = AnalyzeTextSerializer( data=request.data) # pragma: no cover serializer.is_valid(raise_exception=True) # pragma: no cover - request = Repository.request_nlp_analyze( + request = repository.request_nlp_analyze( user_authorization, serializer.data) # pragma: no cover @@ -641,7 +641,7 @@ def evaluate(self, request, **kwargs): detail=_('You need to have at least ' + 'two registered intents')) # pragma: no cover - request = Repository.request_nlp_evaluate( # pragma: no cover + request = repository.request_nlp_evaluate( # pragma: no cover user_authorization, serializer.data) if request.status_code != status.HTTP_200_OK: # pragma: no cover raise APIException( # pragma: no cover diff --git a/bothub/api/v2/repository/views.py b/bothub/api/v2/repository/views.py index 0d1bb80a..4214cba7 100644 --- a/bothub/api/v2/repository/views.py +++ b/bothub/api/v2/repository/views.py @@ -106,7 +106,7 @@ def train(self, request, **kwargs): user_authorization = repository.get_user_authorization(request.user) if not user_authorization.can_write: raise PermissionDenied() - request = Repository.request_nlp_train( # pragma: no cover + request = repository.request_nlp_train( # pragma: no cover user_authorization) if request.status_code != status.HTTP_200_OK: # pragma: no cover raise APIException( # pragma: no cover @@ -126,7 +126,7 @@ def analyze(self, request, **kwargs): serializer = AnalyzeTextSerializer( data=request.data) # pragma: no cover serializer.is_valid(raise_exception=True) # pragma: no cover - request = Repository.request_nlp_analyze( + request = repository.request_nlp_analyze( user_authorization, serializer.data) # pragma: no cover @@ -176,7 +176,7 @@ def evaluate(self, request, **kwargs): detail=_('You need to have at least ' + 'two registered intents')) # pragma: no cover - request = Repository.request_nlp_evaluate( # pragma: no cover + request = repository.request_nlp_evaluate( # pragma: no cover user_authorization, serializer.data) if request.status_code != status.HTTP_200_OK: # pragma: no cover raise APIException( # pragma: no cover diff --git a/bothub/common/models.py b/bothub/common/models.py index bb15aee9..846a500b 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -191,21 +191,13 @@ class Meta: objects = RepositoryManager() - nlp_train_url = '{}train/'.format( - nlp_server if nlp_server else settings.BOTHUB_NLP_BASE_URL - ) - nlp_analyze_url = '{}parse/'.format( - nlp_server if nlp_server else settings.BOTHUB_NLP_BASE_URL - ) - nlp_evaluate_url = '{}evaluate/'.format( - nlp_server if nlp_server else settings.BOTHUB_NLP_BASE_URL - ) - - @classmethod - def request_nlp_train(cls, user_authorization): + def request_nlp_train(self, user_authorization): try: # pragma: no cover r = requests.post( # pragma: no cover - cls.nlp_train_url, + '{}train/'.format( + self.nlp_server if self.nlp_server else + settings.BOTHUB_NLP_BASE_URL + ), data={}, headers={'Authorization': 'Bearer {}'.format( user_authorization.uuid)}) @@ -215,11 +207,13 @@ def request_nlp_train(cls, user_authorization): {'status_code': status.HTTP_503_SERVICE_UNAVAILABLE}, code=status.HTTP_503_SERVICE_UNAVAILABLE) - @classmethod - def request_nlp_analyze(cls, user_authorization, data): + def request_nlp_analyze(self, user_authorization, data): try: # pragma: no cover r = requests.post( # pragma: no cover - cls.nlp_analyze_url, + '{}parse/'.format( + self.nlp_server if self.nlp_server else + settings.BOTHUB_NLP_BASE_URL + ), data={ 'text': data.get('text'), 'language': data.get('language'), @@ -232,11 +226,13 @@ def request_nlp_analyze(cls, user_authorization, data): {'status_code': status.HTTP_503_SERVICE_UNAVAILABLE}, code=status.HTTP_503_SERVICE_UNAVAILABLE) - @classmethod - def request_nlp_evaluate(cls, user_authorization, data): + def request_nlp_evaluate(self, user_authorization, data): try: # pragma: no cover r = requests.post( # pragma: no cover - cls.nlp_evaluate_url, + '{}evaluate/'.format( + self.nlp_server if self.nlp_server else + settings.BOTHUB_NLP_BASE_URL + ), data={ 'language': data.get('language'), }, From 3c9cfdb00933b31c64e691ebf43eb5a67d393520 Mon Sep 17 00:00:00 2001 From: Daniel Yohan Date: Wed, 18 Sep 2019 17:06:35 -0300 Subject: [PATCH 06/10] Update environment and added package environ --- Pipfile | 3 +- Pipfile.lock | 43 +++++++-------- bothub/health/checks.py | 6 +-- bothub/settings.py | 114 ++++++++++++++++++++-------------------- setup.py | 3 +- 5 files changed, 80 insertions(+), 89 deletions(-) diff --git a/Pipfile b/Pipfile index 3c0f9de8..7e02a3a4 100644 --- a/Pipfile +++ b/Pipfile @@ -5,8 +5,6 @@ name = "pypi" [packages] django = "==2.1.5" -dj-database-url = "==0.5.0" -python-decouple = "==3.1" djangorestframework = "==3.9.0" django-filter = "==2.0.0" django-cors-headers = "==2.4.0" @@ -18,6 +16,7 @@ drf-yasg = "*" gunicorn = "*" gevent = "*" packaging = "*" +django-environ = "*" [dev-packages] "flake8" = "*" diff --git a/Pipfile.lock b/Pipfile.lock index c4ab89ec..c0c8806c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5f230c8e911d5f0f91321351555b3277e4a6ce71b6fe1843271cefaf6f11559e" + "sha256": "410e323e9fac34de17c2c5f2db2d94b443a5c717ca7e73226ce764c88280eddb" }, "pipfile-spec": 6, "requires": { @@ -25,10 +25,10 @@ }, "certifi": { "hashes": [ - "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", - "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695" + "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", + "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" ], - "version": "==2019.6.16" + "version": "==2019.9.11" }, "chardet": { "hashes": [ @@ -52,14 +52,6 @@ ], "version": "==0.0.4" }, - "dj-database-url": { - "hashes": [ - "sha256:4aeaeb1f573c74835b0686a2b46b85990571159ffc21aa57ecd4d1e1cb334163", - "sha256:851785365761ebe4994a921b433062309eb882fedd318e1b0fcecc607ed02da9" - ], - "index": "pypi", - "version": "==0.5.0" - }, "django": { "hashes": [ "sha256:a32c22af23634e1d11425574dce756098e015a165be02e4690179889b207c7a8", @@ -76,6 +68,14 @@ "index": "pypi", "version": "==2.4.0" }, + "django-environ": { + "hashes": [ + "sha256:6c9d87660142608f63ec7d5ce5564c49b603ea8ff25da595fd6098f6dc82afde", + "sha256:c57b3c11ec1f319d9474e3e5a79134f40174b17c7cc024bbb2fad84646b120c4" + ], + "index": "pypi", + "version": "==0.4.5" + }, "django-filter": { "hashes": [ "sha256:6f4e4bc1a11151178520567b50320e5c32f8edb552139d93ea3e30613b886f56", @@ -236,13 +236,6 @@ ], "version": "==2.4.2" }, - "python-decouple": { - "hashes": [ - "sha256:1317df14b43efee4337a4aa02914bf004f010cd56d6c4bd894e6474ec8c4fe2d" - ], - "index": "pypi", - "version": "==3.1" - }, "pytz": { "hashes": [ "sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca", @@ -261,10 +254,10 @@ }, "ruamel.yaml": { "hashes": [ - "sha256:547aeab5c51c93bc750ed2a320c1559b605bde3aa569216aa75fd91d8a1c4623", - "sha256:c5e239b6a4f26baabb2e22b145582a7d99ae9d4ebb8902291365a61ed38faa7f" + "sha256:0db639b1b2742dae666c6fc009b8d1931ef15c9276ef31c0673cc6dcf766cf40", + "sha256:412a6f5cfdc0525dee6a27c08f5415c7fd832a7afcb7a0ed7319628aed23d408" ], - "version": "==0.16.1" + "version": "==0.16.5" }, "ruamel.yaml.clib": { "hashes": [ @@ -406,11 +399,11 @@ }, "ipython": { "hashes": [ - "sha256:1d3a1692921e932751bc1a1f7bb96dc38671eeefdc66ed33ee4cbc57e92a410e", - "sha256:537cd0176ff6abd06ef3e23f2d0c4c2c8a4d9277b7451544c6cbf56d1c79a83d" + "sha256:c4ab005921641e40a68e405e286e7a1fcc464497e14d81b6914b4fd95e5dee9b", + "sha256:dd76831f065f17bddd7eaa5c781f5ea32de5ef217592cf019e34043b56895aa1" ], "index": "pypi", - "version": "==7.7.0" + "version": "==7.8.0" }, "ipython-genutils": { "hashes": [ diff --git a/bothub/health/checks.py b/bothub/health/checks.py index 8e147b28..5eb753cd 100644 --- a/bothub/health/checks.py +++ b/bothub/health/checks.py @@ -1,14 +1,12 @@ import logging -from decouple import config +from ..settings import env from rest_framework import status logger = logging.getLogger('bothub.health.checks') -CHECK_ACCESSIBLE_API_URL = config( - 'CHECK_ACCESSIBLE_API_URL', - default=None) +CHECK_ACCESSIBLE_API_URL = env.str('CHECK_ACCESSIBLE_API_URL') def check_database_connection(**kwargs): diff --git a/bothub/settings.py b/bothub/settings.py index 9d006705..b663723f 100644 --- a/bothub/settings.py +++ b/bothub/settings.py @@ -1,27 +1,55 @@ import os -import dj_database_url -from decouple import config +import environ + from django.utils.log import DEFAULT_LOGGING from .utils import cast_supported_languages from .utils import cast_empty_str_to_none +environ.Env.read_env(env_file=(environ.Path(__file__) - 2)('.env')) + +env = environ.Env( + # set casting, default value + DEBUG=(bool, False), + ALLOWED_HOSTS=(lambda v: [s.strip() for s in v.split(',')], '*'), + LANGUAGE_CODE=(str, 'en-us'), + TIME_ZONE=(str, 'UTC'), + STATIC_URL=(str, '/static/'), + EMAIL_HOST=(cast_empty_str_to_none, None), + ADMINS=(lambda v: [ + ( + s.strip().split('|')[0], + s.strip().split('|')[1], + ) for s in v.split(',')] if v else [], []), + DEFAULT_FROM_EMAIL=(str, 'webmaster@localhost'), + SERVER_EMAIL=(str, 'root@localhost'), + EMAIL_PORT=(int, 25), + EMAIL_HOST_USER=(str, ''), + EMAIL_HOST_PASSWORD=(str, ''), + EMAIL_USE_SSL=(bool, False), + EMAIL_USE_TLS=(bool, False), + SEND_EMAILS=(bool, True), + BOTHUB_WEBAPP_BASE_URL=(str, 'http://localhost:8080/'), + BOTHUB_NLP_BASE_URL=(str, 'http://localhost:2657/'), + CSRF_COOKIE_DOMAIN=(cast_empty_str_to_none, None), + CSRF_COOKIE_SECURE=(bool, False), + SUPPORTED_LANGUAGES=(cast_supported_languages, 'en|pt'), + CHECK_ACCESSIBLE_API_URL=(str, None) +) + # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = config('SECRET_KEY') +SECRET_KEY = env.str('SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = config('DEBUG', default=False, cast=bool) +DEBUG = env.bool('DEBUG') -ALLOWED_HOSTS = config( - 'ALLOWED_HOSTS', - default='*', - cast=lambda v: [s.strip() for s in v.split(',')]) +ALLOWED_HOSTS = env.list('ALLOWED_HOSTS') # Application definition @@ -78,11 +106,10 @@ # Database -DATABASES = {} -DATABASES['default'] = dj_database_url.parse( - config( - 'DEFAULT_DATABASE', - default='sqlite:///db.sqlite3')) + +DATABASES = { + 'default': env.db(var='DEFAULT_DATABASE', default='sqlite:///db.sqlite3') +} # Auth @@ -114,9 +141,9 @@ # Internationalization -LANGUAGE_CODE = config('LANGUAGE_CODE', default='en-us') +LANGUAGE_CODE = env.str('LANGUAGE_CODE') -TIME_ZONE = config('TIME_ZONE', default='UTC') +TIME_ZONE = env.str('TIME_ZONE') USE_I18N = True @@ -127,7 +154,7 @@ # Static files (CSS, JavaScript, Images) -STATIC_URL = config('STATIC_URL', default='/static/') +STATIC_URL = env.str('STATIC_URL') STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles') @@ -157,63 +184,41 @@ # mail -envvar_EMAIL_HOST = config( - 'EMAIL_HOST', - default=None, - cast=cast_empty_str_to_none) +envvar_EMAIL_HOST = env.str('EMAIL_HOST') -ADMINS = config( - 'ADMINS', - default='', - cast=lambda v: [ - ( - s.strip().split('|')[0], - s.strip().split('|')[1], - ) for s in v.split(',')] if v else []) +ADMINS = env.list('ADMINS') EMAIL_SUBJECT_PREFIX = '[bothub] ' -DEFAULT_FROM_EMAIL = config( - 'DEFAULT_FROM_EMAIL', - default='webmaster@localhost') -SERVER_EMAIL = config('SERVER_EMAIL', default='root@localhost') +DEFAULT_FROM_EMAIL = env.str('DEFAULT_FROM_EMAIL') +SERVER_EMAIL = env.str('SERVER_EMAIL') if envvar_EMAIL_HOST: EMAIL_HOST = envvar_EMAIL_HOST - EMAIL_PORT = config('EMAIL_PORT', default=25, cast=int) - EMAIL_HOST_USER = config('EMAIL_HOST_USER', default='') - EMAIL_HOST_PASSWORD = config('EMAIL_HOST_PASSWORD', default='') - EMAIL_USE_SSL = config('EMAIL_USE_SSL', default=False, cast=bool) - EMAIL_USE_TLS = config('EMAIL_USE_TLS', default=False, cast=bool) + EMAIL_PORT = env.int('EMAIL_PORT') + EMAIL_HOST_USER = env.str('EMAIL_HOST_USER') + EMAIL_HOST_PASSWORD = env.str('EMAIL_HOST_PASSWORD') + EMAIL_USE_SSL = env.bool('EMAIL_USE_SSL') + EMAIL_USE_TLS = env.bool('EMAIL_USE_TLS') else: EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' -SEND_EMAILS = config('SEND_EMAILS', default=True, cast=bool) +SEND_EMAILS = env.bool('SEND_EMAILS') # webapp -BOTHUB_WEBAPP_BASE_URL = config( - 'BOTHUB_WEBAPP_BASE_URL', - default='http://localhost:8080/') +BOTHUB_WEBAPP_BASE_URL = env.str('BOTHUB_WEBAPP_BASE_URL') # NLP -BOTHUB_NLP_BASE_URL = config( - 'BOTHUB_NLP_BASE_URL', - default='http://localhost:2657/') +BOTHUB_NLP_BASE_URL = env.str('BOTHUB_NLP_BASE_URL') # CSRF -CSRF_COOKIE_DOMAIN = config( - 'CSRF_COOKIE_DOMAIN', - default=None, - cast=cast_empty_str_to_none) +CSRF_COOKIE_DOMAIN = env.str('CSRF_COOKIE_DOMAIN') -CSRF_COOKIE_SECURE = config( - 'CSRF_COOKIE_SECURE', - default=False, - cast=bool) +CSRF_COOKIE_SECURE = env.bool('CSRF_COOKIE_SECURE') # Logging @@ -236,10 +241,7 @@ # Supported Languages -SUPPORTED_LANGUAGES = config( - 'SUPPORTED_LANGUAGES', - default='en|pt', - cast=cast_supported_languages) +SUPPORTED_LANGUAGES = env.str('SUPPORTED_LANGUAGES') # SECURE PROXY SSL HEADER diff --git a/setup.py b/setup.py index 75f37617..0420107a 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,7 @@ packages=find_packages(), install_requires=[ 'django==2.1.3', - 'dj-database-url==0.5.0', - 'python-decouple==3.1', + 'django-environ==0.4.5', 'djangorestframework==3.9.0', 'django-filter==2.0.0', 'django-cors-headers==2.4.0', From 0b438e8310143d031cdc22735665bf6279f3e274 Mon Sep 17 00:00:00 2001 From: Daniel Yohan Date: Thu, 19 Sep 2019 08:45:19 -0300 Subject: [PATCH 07/10] [fix] Variable Environment --- bothub/settings.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bothub/settings.py b/bothub/settings.py index b663723f..92556ab8 100644 --- a/bothub/settings.py +++ b/bothub/settings.py @@ -241,7 +241,12 @@ # Supported Languages -SUPPORTED_LANGUAGES = env.str('SUPPORTED_LANGUAGES') +SUPPORTED_LANGUAGES = env.get_value( + 'SUPPORTED_LANGUAGES', + cast_supported_languages, + 'en|pt', + True +) # SECURE PROXY SSL HEADER From 525cfe114eb25bebcba45e74f7c6cf013d8734be Mon Sep 17 00:00:00 2001 From: Daniel Yohan Date: Thu, 19 Sep 2019 09:25:12 -0300 Subject: [PATCH 08/10] [fix] Variable Environment --- bothub/settings.py | 6 +++++- bothub/utils.py | 11 +++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/bothub/settings.py b/bothub/settings.py index 92556ab8..58df4822 100644 --- a/bothub/settings.py +++ b/bothub/settings.py @@ -36,7 +36,11 @@ CSRF_COOKIE_DOMAIN=(cast_empty_str_to_none, None), CSRF_COOKIE_SECURE=(bool, False), SUPPORTED_LANGUAGES=(cast_supported_languages, 'en|pt'), - CHECK_ACCESSIBLE_API_URL=(str, None) + CHECK_ACCESSIBLE_API_URL=(str, None), + BOTHUB_ENGINE_AWS_ACCESS_KEY_ID=(str, ''), + BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY=(str, ''), + BOTHUB_ENGINE_AWS_S3_BUCKET_NAME=(str, ''), + BOTHUB_ENGINE_AWS_REGION_NAME=(str, 'us-east-1') ) # Build paths inside the project like this: os.path.join(BASE_DIR, ...) diff --git a/bothub/utils.py b/bothub/utils.py index e3b681cd..0ab597fc 100644 --- a/bothub/utils.py +++ b/bothub/utils.py @@ -1,7 +1,7 @@ import io import uuid import boto3 -from decouple import config +from bothub.settings import env from botocore.exceptions import ClientError from collections import OrderedDict @@ -18,11 +18,10 @@ def cast_empty_str_to_none(value): def send_bot_data_file_aws(id, bot_data): - aws_access_key_id = config('BOTHUB_ENGINE_AWS_ACCESS_KEY_ID', default='') - aws_secret_access_key = config( - 'BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY', default='') - aws_bucket_name = config('BOTHUB_ENGINE_AWS_S3_BUCKET_NAME', default='') - aws_region_name = config('BOTHUB_ENGINE_AWS_REGION_NAME', 'us-east-1') + aws_access_key_id = env.str('BOTHUB_ENGINE_AWS_ACCESS_KEY_ID') + aws_secret_access_key = env.str('BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY') + aws_bucket_name = env.str('BOTHUB_ENGINE_AWS_S3_BUCKET_NAME') + aws_region_name = env.str('BOTHUB_ENGINE_AWS_REGION_NAME') confmat_url = '' From 77419e43ecca30ac594569bed4febcbf13818c4b Mon Sep 17 00:00:00 2001 From: Daniel Yohan Date: Thu, 19 Sep 2019 09:28:26 -0300 Subject: [PATCH 09/10] [fix] Pipfile --- Pipfile.lock | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index a2724b02..2cf3a28c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "410e323e9fac34de17c2c5f2db2d94b443a5c717ca7e73226ce764c88280eddb" + "sha256": "763e79ede86d704b4ed149cd4146aa1539d334d3c949f5fc570b5c45bf431069" }, "pipfile-spec": 6, "requires": { @@ -16,27 +16,20 @@ ] }, "default": { - "attrs": { - "hashes": [ - "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", - "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" - ], - "version": "==19.1.0" - }, "boto3": { "hashes": [ - "sha256:654c7ebd6d089d5af634a8121f3960e50e283643660abcba07e602ac237f4839", - "sha256:f114b586c307f73a46d6dfe9dfb1c37865354f48fc749794d96517527424d1f5" + "sha256:5d2665d1407c6dd932f609d72233b567a9959bf1409fc0e1b79ea6ff00f97324", + "sha256:b34199b188c03d233f23b39f1149d78e0edda6f69da09de9c46ba67f2850cec6" ], "index": "pypi", - "version": "==1.9.220" + "version": "==1.9.231" }, "botocore": { "hashes": [ - "sha256:748fe4ee5cc8b10ef09e52c740b488402d6f6d4d1f0dde0c936da232b42b1bdd", - "sha256:9ffd9264e4ad999d2929cfe1c7e413d4cdf76a8bd92f011dce31874f056d2e18" + "sha256:535d8a81418d7cac9de83abac1733c4e79dc5d0645c884c0d680c72df2d7825a", + "sha256:b9d0248ca2dae2b15a29f2d29d81175072a3ffb48056238d6d9584597810878d" ], - "version": "==1.12.220" + "version": "==1.12.231" }, "certifi": { "hashes": [ @@ -253,11 +246,11 @@ }, "packaging": { "hashes": [ - "sha256:a7ac867b97fdc07ee80a8058fe4435ccd274ecc3b0ed61d852d7d53055528cf9", - "sha256:c491ca87294da7cc01902edbe30a5bc6c4c28172b5138ab4e4aa1b9d7bfaeafe" + "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", + "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" ], "index": "pypi", - "version": "==19.1" + "version": "==19.2" }, "pyparsing": { "hashes": [ @@ -266,6 +259,14 @@ ], "version": "==2.4.2" }, + "python-dateutil": { + "hashes": [ + "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", + "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + ], + "markers": "python_version >= '2.7'", + "version": "==2.8.0" + }, "pytz": { "hashes": [ "sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca", From dc56ed75221df57b5c63e3f23885aebd0937b154 Mon Sep 17 00:00:00 2001 From: Daniel Yohan Date: Thu, 19 Sep 2019 09:32:34 -0300 Subject: [PATCH 10/10] [fix] Variable Environment --- bothub/settings.py | 8 ++++++++ bothub/utils.py | 20 ++++++++------------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/bothub/settings.py b/bothub/settings.py index 58df4822..0f908f4b 100644 --- a/bothub/settings.py +++ b/bothub/settings.py @@ -272,3 +272,11 @@ }, }, } + + +# AWS + +AWS_ACCESS_KEY_ID = env.str('BOTHUB_ENGINE_AWS_ACCESS_KEY_ID') +AWS_SECRET_ACCESS_KEY = env.str('BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY') +AWS_BUCKET_NAME = env.str('BOTHUB_ENGINE_AWS_S3_BUCKET_NAME') +AWS_REGION_NAME = env.str('BOTHUB_ENGINE_AWS_REGION_NAME') diff --git a/bothub/utils.py b/bothub/utils.py index 0ab597fc..4f0e2259 100644 --- a/bothub/utils.py +++ b/bothub/utils.py @@ -1,7 +1,7 @@ import io import uuid import boto3 -from bothub.settings import env +from django.conf import settings from botocore.exceptions import ClientError from collections import OrderedDict @@ -18,14 +18,10 @@ def cast_empty_str_to_none(value): def send_bot_data_file_aws(id, bot_data): - aws_access_key_id = env.str('BOTHUB_ENGINE_AWS_ACCESS_KEY_ID') - aws_secret_access_key = env.str('BOTHUB_ENGINE_AWS_SECRET_ACCESS_KEY') - aws_bucket_name = env.str('BOTHUB_ENGINE_AWS_S3_BUCKET_NAME') - aws_region_name = env.str('BOTHUB_ENGINE_AWS_REGION_NAME') - confmat_url = '' - if all([aws_access_key_id, aws_secret_access_key, aws_bucket_name]): + if all([settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY, + settings.AWS_BUCKET_NAME]): confmat_filename = \ f'repository_{str(id)}/bot_data_{uuid.uuid4()}.tar.gz' @@ -33,20 +29,20 @@ def send_bot_data_file_aws(id, bot_data): s3_client = boto3.client( 's3', - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - region_name=aws_region_name, + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + region_name=settings.AWS_REGION_NAME, ) try: s3_client.upload_fileobj( botdata, - aws_bucket_name, + settings.AWS_BUCKET_NAME, confmat_filename, ExtraArgs={'ContentType': 'application/gzip'} ) confmat_url = '{}/{}/{}'.format( s3_client.meta.endpoint_url, - aws_bucket_name, + settings.AWS_BUCKET_NAME, confmat_filename ) except ClientError as e: