Develop (#709) (#710)

* Fix: Remove ai from project (#707)

* remove document deletion from delete_nlp_logs task

* inconsistency number and debug errors fixed

* add user_email to remove_authorizations_project

* Feature/health check blocklist (#708)

* remove document deletion from delete_nlp_logs task

* add a blocklist for not saving logs depending on the authorization user

* inconsistency number and debug errors fixed

* change the REPOSITORY_BLOCK_USER_LOGS values from users to repository authorizations

* change readme

* pass on sonarcloud

* change admins settings

* transform uuid into string

* convert uuid into string at test_blocked_user

README.md

@@ -181,6 +181,7 @@ You can set environment variables in your OS, write on ```.env``` file or pass v
 | ELASTICSEARCH_LOGS_DELETE_AGE | ```string``` | ```90d``` | Specify the ILM delete age, when the index will be deleted.
 | GUNICORN_WORKERS | ``` int ``` | ``` multiprocessing.cpu_count() * 2 + 1 ``` | Gunicorn number of workers.
 | USE_ELASTICSEARCH | ```boolean``` | ```true``` | Change the logic in requirements_to_train to use either elasticsearch or postgres.
+| REPOSITORY_BLOCK_USER_LOGS | ```list``` | ```[]``` | List of repository authorization(api bearer) that won't save logs
 
 
 ## Roadmap

bothub/api/grpc/connect_grpc_client.py

@@ -57,14 +57,18 @@ def get_authorization_classifier(
 
         return next(classifier).get("uuid")
 
-    def remove_authorization(self, project_uuid: str, authorization_uuid: str):
+    def remove_authorization(
+        self, project_uuid: str, authorization_uuid: str, user_email: str
+    ):
         classifier_uuid = self.get_authorization_classifier(
             project_uuid, authorization_uuid
         )
 
         stub = project_pb2_grpc.ProjectControllerStub(self.channel)
         stub.DestroyClassifier(
-            project_pb2.ClassifierDestroyRequest(uuid=classifier_uuid)
+            project_pb2.ClassifierDestroyRequest(
+                uuid=classifier_uuid, user_email=user_email
+            )
         )
 
     def create_classifier(self, **kwargs):

bothub/api/v2/nlp/serializers.py

@@ -1,5 +1,7 @@
 from rest_framework import serializers
 
+from django.conf import settings
+
 from bothub.common.models import (
     QAKnowledgeBase,
     QALogs,
@@ -45,8 +47,12 @@ class Meta:
     )
 
     def create(self, validated_data):
+        repository_auth = validated_data.get("user")
+        user = repository_auth.user
+        if str(repository_auth.pk) in settings.REPOSITORY_BLOCK_USER_LOGS:
+            return validated_data
         log_intent = validated_data.pop("log_intent")
-        validated_data.update({"user": validated_data.get("user").user})
+        validated_data.update({"user": user})
 
         instance = self.Meta.model(**validated_data)
         instance.save()

bothub/api/v2/repository/views.py

@@ -336,7 +336,7 @@ def remove_repository_project(self, request, **kwargs):
 
         task = celery_app.send_task(
             name="remove_authorizations_project",
-            args=[project_uuid, list(authorizations_uuids)],
+            args=[project_uuid, list(authorizations_uuids), request.user.email],
         )
         task.wait()
 

bothub/api/v2/tests/test_logs.py

@@ -23,6 +23,7 @@
 from bothub.common.documents.repositorynlplog import REPOSITORYNLPLOG_INDEX_NAME
 
 
+@tag("elastic")
 class RepositoryNLPLogTestCase(TestCase):
     def setUp(self):
         self.factory = RequestFactory()
@@ -59,6 +60,58 @@ def request(self, data):
         content_data = json.loads(response.content)
         return (response, content_data)
 
+    def list_request(self, data, token=None):
+        authorization_header = (
+            {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} if token else {}
+        )
+        request = self.factory.get("/v2/repository/log/", data, **authorization_header)
+        response = RepositoryNLPLogViewSet.as_view({"get": "list"})(request)
+        response.render()
+        content_data = json.loads(response.content)
+        return (response, content_data)
+
+    def test_blocked_user(self):
+        with self.settings(REPOSITORY_BLOCK_USER_LOGS=[str(self.repository_auth.pk)]):
+            data = {
+                "text": "test",
+                "user_agent": "python-requests/2.20.1",
+                "from_backend": True,
+                "user": str(self.repository_auth.pk),
+                "repository_version_language": int(
+                    self.repository.current_version().pk
+                ),
+                "nlp_log": json.dumps(
+                    {
+                        "intent": {"name": "bias", "confidence": 0.9994810899625854},
+                        "intent_ranking": [
+                            {"name": "bias", "confidence": 0.9994810819625244},
+                            {"name": "doubt", "confidence": 0.039216167263031006},
+                            {"name": "negative", "confidence": 0.0},
+                            {"name": "affirmative", "confidence": 0.0},
+                        ],
+                        "labels_list": [],
+                        "entities_list": [],
+                        "entities": {},
+                        "text": "test",
+                        "repository_version": int(self.repository.current_version().pk),
+                        "language": str(self.repository.language),
+                    }
+                ),
+                "log_intent": [],
+            }
+            self.request(data)
+            response, content_data = self.list_request(
+                {
+                    "repository_version_language": int(
+                        self.repository.current_version().pk
+                    )
+                },
+                self.owner_token,
+            )
+
+            self.assertEqual(response.status_code, status.HTTP_200_OK)
+            self.assertEqual(content_data.get("count"), 0)
+
     def test_okay(self):
         data = {
             "text": "test",

bothub/common/tasks.py

@@ -551,10 +551,12 @@ def get_project_organization(project_uuid: str):  # pragma: no cover
 
 
 @app.task(name="remove_authorizations_project")
-def remove_authorizations_project(project_uuid: str, authorizations_uuids: list):
+def remove_authorizations_project(
+    project_uuid: str, authorizations_uuids: list, user_email: str
+):
     grpc_client = ConnectGRPCClient()
     for authorization_uuid in authorizations_uuids:
-        grpc_client.remove_authorization(project_uuid, authorization_uuid)
+        grpc_client.remove_authorization(project_uuid, authorization_uuid, user_email)
 
 
 @app.task(name="create_repository_project")

bothub/settings.py

@@ -76,6 +76,7 @@
     CONNECT_GRPC_SERVER_URL=(str, "localhost:8002"),
     CONNECT_CERTIFICATE_GRPC_CRT=(str, None),
     REPOSITORY_RESTRICT_ACCESS_NLP_LOGS=(list, []),
+    REPOSITORY_BLOCK_USER_LOGS=(list, []),
     REPOSITORY_KNOWLEDGE_BASE_DESCRIPTION_LIMIT=(int, 450),
     REPOSITORY_EXAMPLE_TEXT_WORDS_LIMIT=(int, 200),
     ELASTICSEARCH_DSL=(str, "localhost:9200"),
@@ -263,7 +264,7 @@
 
 envvar_EMAIL_HOST = env.str("EMAIL_HOST")
 
-ADMINS = env.list("ADMINS")
+ADMINS = [("Helder", "helder.souza@weni.ai")]  # env.list("ADMINS")
 EMAIL_SUBJECT_PREFIX = "[bothub] "
 DEFAULT_FROM_EMAIL = env.str("DEFAULT_FROM_EMAIL")
 SERVER_EMAIL = env.str("SERVER_EMAIL")
@@ -451,12 +452,12 @@
 REPOSITORY_RESTRICT_ACCESS_NLP_LOGS = env.list("REPOSITORY_RESTRICT_ACCESS_NLP_LOGS")
 
 # Limit of characters for the knowledge base description
-REPOSITORY_KNOWLEDGE_BASE_DESCRIPTION_LIMIT = env.list(
+REPOSITORY_KNOWLEDGE_BASE_DESCRIPTION_LIMIT = env.int(
     "REPOSITORY_KNOWLEDGE_BASE_DESCRIPTION_LIMIT", default=450
 )
 
 # Limit of words for the example sentence
-REPOSITORY_EXAMPLE_TEXT_WORDS_LIMIT = env.list(
+REPOSITORY_EXAMPLE_TEXT_WORDS_LIMIT = env.int(
     "REPOSITORY_EXAMPLE_TEXT_WORDS_LIMIT", default=200
 )
 
@@ -582,3 +583,5 @@
 ELASTICSEARCH_DSL_SIGNAL_PROCESSOR = ELASTICSEARCH_SIGNAL_PROCESSOR_CLASSES[
     env.str("ELASTICSEARCH_SIGNAL_PROCESSOR", default="realtime")
 ]
+
+REPOSITORY_BLOCK_USER_LOGS = env.list("REPOSITORY_BLOCK_USER_LOGS", default=[])

docker-compose.yml

@@ -7,16 +7,24 @@ version: '3.6'
 
 services:
   database:
-      image: postgres
-      ports:
-        - 5432:5432
-      networks:
-        - default
-        - bothub
-      environment:
-        - POSTGRES_USER=${POSTGRES_USER:-bothub}
-        - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-bothub}
-        - POSTGRES_DB=${POSTGRES_DB:-bothub}
+    image: postgres
+    ports:
+      - 5432:5432
+    volumes:
+      - postgres:/var/lib/postgresql/data
+    networks:
+      - default
+      - bothub
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER:-bothub}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-bothub}
+      - POSTGRES_DB=${POSTGRES_DB:-bothub}
+    deploy:
+      mode: replicated
+      replicas: 1
+      placement:
+        constraints: [node.role == manager]
+
   bothub:
     image: ${DOCKER_IMAGE_NAME:-ilha/bothub}:${TAG:-latest}
     build:
@@ -153,6 +161,9 @@ services:
       - default
       - bothub
 
+volumes:
+    postgres:
+
 networks:
   bothub:
     external: true