Merge pull request #653 from Ilhasoft/staging

Staging

.flake8

@@ -6,4 +6,3 @@ exclude =
   ./bothub/common/migrations
   ./manage.py
   ./bothub/common/languages.py
-  ./bothub/protos/

.gitignore

@@ -117,9 +117,6 @@ staticfiles/
 #Pycharm
 .idea/
 
-# gRPC Protobuff
-bothub/protos/
-
 #ASDF
 .tool-versions
 

.travis.yml

@@ -57,7 +57,6 @@ env:
     - ELASTICSEARCH_NUMBER_OF_SHARDS=1
     - ELASTICSEARCH_NUMBER_OF_REPLICAS=1
 script:
-  - make createproto
   - python manage.py migrate
   - python manage.py search_index --rebuild -f
   - python manage.py collectstatic

Dockerfile

@@ -25,7 +25,5 @@ RUN pipenv install --system
 
 COPY . .
 
-RUN make createproto
-
 RUN chmod +x ./entrypoint.sh
 ENTRYPOINT [ "./entrypoint.sh" ]

Makefile

@@ -72,16 +72,6 @@ search_index:
 		then python manage.py search_index --rebuild -f; \
 		else pipenv run python manage.py search_index --rebuild -f; fi
 
-createproto:
-
-	@rm -rf ./bothub/protos/
-	@git clone --depth 1 --branch main https://github.com/Ilhasoft/weni-protobuffers ./bothub/protos/
-	@python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/src/weni/protobuf/intelligence/authentication.proto
-	@python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/src/weni/protobuf/intelligence/organization.proto
-	@python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/src/weni/protobuf/intelligence/repository.proto	
-	@python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/src/weni/protobuf/connect/project.proto
-
-
 # Utils
 
 ## Colors

Pipfile

@@ -40,6 +40,7 @@ django-elasticsearch-dsl-drf= "~=0.22.1"
 elasticsearch= "~=7.14.0"
 elasticsearch-dsl= "~=7.4.0"
 psycopg2-binary = "~=2.9.1"
+weni-protobuffers = "~=1.1.0"
 
 [dev-packages]
 "flake8" = "~=3.9.2"

README.md

@@ -163,6 +163,7 @@ You can set environment variables in your OS, write on ```.env``` file or pass v
 | ELASTICSEARCH_NUMBER_OF_REPLICAS | ```int``` | ```1``` | Specify the number of replicas for the indexes.
 | ELASTICSEARCH_REPOSITORYNLPLOG_INDEX | ```string``` | ```repositorynlplog``` | Specify the index title for the RepositoryNlpLog document.
 | ELASTICSEARCH_SIGNAL_PROCESSOR | ```string``` | ```celery``` | Specify the signal processor responsible for updating the Elasticsearch data.
+| GUNICORN_WORKERS | ``` int ``` | ``` multiprocessing.cpu_count() * 2 + 1 ``` | Gunicorn number of workers
 
 
 ## Roadmap

bothub/api/grpc/connect_grpc_client.py

@@ -3,7 +3,7 @@
 import grpc
 from django.conf import settings
 
-from bothub.protos.src.weni.protobuf.connect import project_pb2_grpc, project_pb2
+from weni.protobuf.connect import project_pb2_grpc, project_pb2
 
 
 class ConnectGRPCClient:

bothub/api/grpc/organization/handlers.py

@@ -1,5 +1,5 @@
 from bothub.api.grpc.organization.services import OrgService
-from bothub.protos.src.weni.protobuf.intelligence import organization_pb2_grpc
+from weni.protobuf.intelligence import organization_pb2_grpc
 
 
 def grpc_handlers(server):

bothub/api/grpc/organization/serializers.py

@@ -4,7 +4,7 @@
 
 from bothub.authentication.models import User
 from bothub.common.models import Organization, OrganizationAuthorization
-from bothub.protos.src.weni.protobuf.intelligence import organization_pb2
+from weni.protobuf.intelligence import organization_pb2
 
 
 class OrgProtoSerializer(proto_serializers.ModelProtoSerializer):

bothub/api/grpc/organization/services.py

@@ -16,7 +16,7 @@
     RepositoryAuthorization,
     Repository,
 )
-from bothub.protos.src.weni.protobuf.intelligence.organization_pb2 import OrgStatistic
+from weni.protobuf.intelligence.organization_pb2 import OrgStatistic
 
 
 class OrgService(

bothub/api/grpc/repository/handlers.py

@@ -1,5 +1,5 @@
 from .services import RepositoryService
-from bothub.protos.src.weni.protobuf.intelligence import repository_pb2_grpc
+from weni.protobuf.intelligence import repository_pb2_grpc
 
 
 def grpc_handlers(server):

bothub/api/grpc/repository/serializers.py

@@ -3,7 +3,7 @@
 
 from bothub.api.v2.repository.serializers import RepositoryCategorySerializer
 from bothub.common.models import Repository
-from bothub.protos.src.weni.protobuf.intelligence import repository_pb2
+from weni.protobuf.intelligence import repository_pb2
 
 
 class RepositoryProtoSerializer(proto_serializers.ModelProtoSerializer):

bothub/api/grpc/repository/tests.py

@@ -8,7 +8,7 @@
     OrganizationAuthorization,
     RepositoryAuthorization,
 )
-from bothub.protos.src.weni.protobuf.intelligence import (
+from weni.protobuf.intelligence import (
     repository_pb2_grpc,
     repository_pb2,
 )

bothub/api/grpc/user/handlers.py

@@ -3,7 +3,7 @@
     UserService,
     UserLanguageService,
 )
-from bothub.protos.src.weni.protobuf.intelligence import authentication_pb2_grpc
+from weni.protobuf.intelligence import authentication_pb2_grpc
 
 
 def grpc_handlers(server):

bothub/api/grpc/user/serializers.py

@@ -2,7 +2,7 @@
 from rest_framework import serializers
 
 from bothub.authentication.models import User
-from bothub.protos.src.weni.protobuf.intelligence import authentication_pb2
+from weni.protobuf.intelligence import authentication_pb2
 
 
 class UserProtoSerializer(proto_serializers.ModelProtoSerializer):

bothub/api/grpc/user/tests.py

@@ -3,7 +3,7 @@
 from bothub.api.v2.tests.utils import create_user_and_token
 from bothub.authentication.models import User
 from bothub.common.models import Organization, OrganizationAuthorization
-from bothub.protos.src.weni.protobuf.intelligence import (
+from weni.protobuf.intelligence import (
     authentication_pb2_grpc,
     authentication_pb2,
 )
@@ -25,8 +25,8 @@ def setUp(self):
 
         super().setUp()
 
-        self.user_permission_stub = authentication_pb2_grpc.UserPermissionControllerStub(
-            self.channel
+        self.user_permission_stub = (
+            authentication_pb2_grpc.UserPermissionControllerStub(self.channel)
         )
         self.user_stub = authentication_pb2_grpc.UserControllerStub(self.channel)
 

bothub/api/v2/knowledge_base/filters.py

@@ -3,7 +3,7 @@
 from django_filters import rest_framework as filters
 from rest_framework.exceptions import PermissionDenied, NotFound
 
-from bothub.common.models import QAKnowledgeBase, Repository, QAContext
+from bothub.common.models import QAKnowledgeBase, Repository, QAtext
 
 
 class QAKnowledgeBaseFilter(filters.FilterSet):
@@ -34,9 +34,9 @@ def filter_repository_uuid(self, queryset, name, value):
             raise NotFound(_("Invalid repository_uuid"))
 
 
-class QAContextFilter(filters.FilterSet):
+class QAtextFilter(filters.FilterSet):
     class Meta:
-        model = QAContext
+        model = QAtext
         fields = [
             "text",
             "language",

bothub/api/v2/knowledge_base/permissions.py

@@ -3,20 +3,17 @@
 from bothub.api.v2 import READ_METHODS
 
 
-class QAKnowledgeBasePermission(permissions.BasePermission):
+class QABasePermission(permissions.BasePermission):
     def has_object_permission(self, request, view, obj):
-        authorization = obj.repository.get_user_authorization(request.user)
+        authorization = obj.get_user_authorization(request.user)
         if request.method in READ_METHODS:
             return authorization.can_read
         return authorization.can_contribute
 
 
-class QAContextPermission(permissions.BasePermission):
-    def has_object_permission(self, request, view, obj):
-        authorization = obj.knowledge_base.repository.get_user_authorization(
-            request.user
-        )
-        if request.method in READ_METHODS:
-            return authorization.can_read
+class QAKnowledgeBasePermission(QABasePermission):
+    pass
 
-        return authorization.can_contribute
+
+class QAtextPermission(QABasePermission):
+    pass

bothub/api/v2/knowledge_base/serializers.py

@@ -1,25 +1,60 @@
 from rest_framework import serializers
 
 from bothub.common import languages
-from bothub.common.models import Repository, QAKnowledgeBase, QAContext
+from bothub.common.models import Repository, QAKnowledgeBase, QAtext
 
 
 class QAKnowledgeBaseSerializer(serializers.ModelSerializer):
     class Meta:
         model = QAKnowledgeBase
-        fields = ["id", "repository", "title"]
+        fields = [
+            "id",
+            "repository",
+            "title",
+            "description",
+            "language_count",
+            "user_name",
+            "user",
+            "last_update",
+            "created_at",
+        ]
         read_only_fields = ["created_at", "last_update"]
 
+    user = serializers.HiddenField(default=serializers.CurrentUserDefault())
+    user_name = serializers.SerializerMethodField("get_user_name")
     repository = serializers.PrimaryKeyRelatedField(queryset=Repository.objects)
+    description = serializers.SerializerMethodField("get_description")
+    language_count = serializers.SerializerMethodField("get_languages_count")
 
+    def get_user_name(self, obj):
+        return None if obj.user is None else obj.user.nickname
 
-class QAContextSerializer(serializers.ModelSerializer):
+    def get_description(self, obj):
+        return obj.get_text_description()
+
+    def get_languages_count(self, obj):
+        return obj.get_languages_count()
+
+
+class QAtextSerializer(serializers.ModelSerializer):
     class Meta:
-        model = QAContext
-        fields = ["id", "text", "language", "knowledge_base"]
+        model = QAtext
+        fields = [
+            "id",
+            "text",
+            "language",
+            "knowledge_base",
+            "created_at",
+            "last_update",
+            "title",
+        ]
         read_only_fields = ["created_at", "last_update"]
 
     knowledge_base = serializers.PrimaryKeyRelatedField(
         queryset=QAKnowledgeBase.objects
     )
     language = serializers.ChoiceField(languages.LANGUAGE_CHOICES, required=True)
+    title = serializers.SerializerMethodField("get_title")
+
+    def get_title(self, obj):
+        return obj.get_title()

bothub/api/v2/knowledge_base/views.py

@@ -1,12 +1,31 @@
+from django.utils.decorators import method_decorator
+
 from rest_framework import mixins, permissions
 from rest_framework.viewsets import GenericViewSet
+from drf_yasg2 import openapi
+from drf_yasg2.utils import swagger_auto_schema
 
-from .filters import QAKnowledgeBaseFilter, QAContextFilter
-from .permissions import QAKnowledgeBasePermission, QAContextPermission
-from .serializers import QAKnowledgeBaseSerializer, QAContextSerializer
-from bothub.common.models import QAKnowledgeBase, QAContext
+from bothub.common.models import QAKnowledgeBase, QAtext
+from .filters import QAKnowledgeBaseFilter, QAtextFilter
+from .permissions import QAKnowledgeBasePermission, QAtextPermission
+from .serializers import QAKnowledgeBaseSerializer, QAtextSerializer
 
 
+@method_decorator(
+    name="list",
+    decorator=swagger_auto_schema(
+        manual_parameters=[
+            openapi.Parameter(
+                "repository_uuid",
+                openapi.IN_QUERY,
+                description="Repository's UUID",
+                required=True,
+                type=openapi.TYPE_STRING,
+                format="uuid",
+            ),
+        ]
+    ),
+)
 class QAKnowledgeBaseViewSet(
     mixins.ListModelMixin,
     mixins.CreateModelMixin,
@@ -21,15 +40,30 @@ class QAKnowledgeBaseViewSet(
     permission_classes = [permissions.IsAuthenticated, QAKnowledgeBasePermission]
 
 
-class QAContextViewSet(
+@method_decorator(
+    name="list",
+    decorator=swagger_auto_schema(
+        manual_parameters=[
+            openapi.Parameter(
+                "repository_uuid",
+                openapi.IN_QUERY,
+                description="Repository's UUID",
+                required=True,
+                type=openapi.TYPE_STRING,
+                format="uuid",
+            ),
+        ]
+    ),
+)
+class QAtextViewSet(
     mixins.ListModelMixin,
     mixins.CreateModelMixin,
     mixins.RetrieveModelMixin,
     mixins.DestroyModelMixin,
     mixins.UpdateModelMixin,
     GenericViewSet,
 ):
-    queryset = QAContext.objects.all()
-    serializer_class = QAContextSerializer
-    filter_class = QAContextFilter
-    permission_classes = [permissions.IsAuthenticated, QAContextPermission]
+    queryset = QAtext.objects.all()
+    serializer_class = QAtextSerializer
+    filter_class = QAtextFilter
+    permission_classes = [permissions.IsAuthenticated, QAtextPermission]

bothub/api/v2/nlp/serializers.py

@@ -1,6 +1,8 @@
 from rest_framework import serializers
 
 from bothub.common.models import (
+    QAKnowledgeBase,
+    QALogs,
     RepositoryNLPLog,
     RepositoryNLPLogIntent,
     RepositoryVersionLanguage,
@@ -58,3 +60,36 @@ def create(self, validated_data):
             )
 
         return instance
+
+
+class RepositoryQANLPLogSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = QALogs
+        fields = [
+            "id",
+            "answer",
+            "confidence",
+            "question",
+            "user_agent",
+            "nlp_log",
+            "user",
+            "knowledge_base",
+            "language",
+            "from_backend",
+        ]
+        ref_name = None
+
+    knowledge_base = serializers.PrimaryKeyRelatedField(
+        queryset=QAKnowledgeBase.objects, write_only=True, required=True
+    )
+    user = serializers.PrimaryKeyRelatedField(
+        queryset=RepositoryAuthorization.objects, write_only=True, required=True
+    )
+
+    def create(self, validated_data):
+        validated_data.update({"user": validated_data.get("user").user})
+
+        instance = self.Meta.model(**validated_data)
+        instance.save()
+
+        return instance