Skip to content

Commit

Permalink
Merge 62eacf7 into ecbe55b
Browse files Browse the repository at this point in the history
  • Loading branch information
helllllllder committed Apr 28, 2022
2 parents ecbe55b + 62eacf7 commit d9f0de7
Show file tree
Hide file tree
Showing 5 changed files with 88 additions and 14 deletions.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@ You can set environment variables in your OS, write on ```.env``` file or pass v
| ELASTICSEARCH_LOGS_DELETE_AGE | ```string``` | ```90d``` | Specify the ILM delete age, when the index will be deleted.
| GUNICORN_WORKERS | ``` int ``` | ``` multiprocessing.cpu_count() * 2 + 1 ``` | Gunicorn number of workers.
| USE_ELASTICSEARCH | ```boolean``` | ```true``` | Change the logic in requirements_to_train to use either elasticsearch or postgres.
| REPOSITORY_BLOCK_USER_LOGS | ```list``` | ```[]``` | List of repository authorization(api bearer) that won't save logs


## Roadmap
Expand Down
8 changes: 7 additions & 1 deletion bothub/api/v2/nlp/serializers.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
from rest_framework import serializers

from django.conf import settings

from bothub.common.models import (
QAKnowledgeBase,
QALogs,
Expand Down Expand Up @@ -45,8 +47,12 @@ class Meta:
)

def create(self, validated_data):
repository_auth = validated_data.get("user")
user = repository_auth.user
if str(repository_auth.pk) in settings.REPOSITORY_BLOCK_USER_LOGS:
return validated_data
log_intent = validated_data.pop("log_intent")
validated_data.update({"user": validated_data.get("user").user})
validated_data.update({"user": user})

instance = self.Meta.model(**validated_data)
instance.save()
Expand Down
53 changes: 53 additions & 0 deletions bothub/api/v2/tests/test_logs.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from bothub.common.documents.repositorynlplog import REPOSITORYNLPLOG_INDEX_NAME


@tag("elastic")
class RepositoryNLPLogTestCase(TestCase):
def setUp(self):
self.factory = RequestFactory()
Expand Down Expand Up @@ -59,6 +60,58 @@ def request(self, data):
content_data = json.loads(response.content)
return (response, content_data)

def list_request(self, data, token=None):
authorization_header = (
{"HTTP_AUTHORIZATION": "Token {}".format(token.key)} if token else {}
)
request = self.factory.get("/v2/repository/log/", data, **authorization_header)
response = RepositoryNLPLogViewSet.as_view({"get": "list"})(request)
response.render()
content_data = json.loads(response.content)
return (response, content_data)

def test_blocked_user(self):
with self.settings(REPOSITORY_BLOCK_USER_LOGS=[str(self.repository_auth.pk)]):
data = {
"text": "test",
"user_agent": "python-requests/2.20.1",
"from_backend": True,
"user": str(self.repository_auth.pk),
"repository_version_language": int(
self.repository.current_version().pk
),
"nlp_log": json.dumps(
{
"intent": {"name": "bias", "confidence": 0.9994810899625854},
"intent_ranking": [
{"name": "bias", "confidence": 0.9994810819625244},
{"name": "doubt", "confidence": 0.039216167263031006},
{"name": "negative", "confidence": 0.0},
{"name": "affirmative", "confidence": 0.0},
],
"labels_list": [],
"entities_list": [],
"entities": {},
"text": "test",
"repository_version": int(self.repository.current_version().pk),
"language": str(self.repository.language),
}
),
"log_intent": [],
}
self.request(data)
response, content_data = self.list_request(
{
"repository_version_language": int(
self.repository.current_version().pk
)
},
self.owner_token,
)

self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(content_data.get("count"), 0)

def test_okay(self):
data = {
"text": "test",
Expand Down
9 changes: 6 additions & 3 deletions bothub/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@
CONNECT_GRPC_SERVER_URL=(str, "localhost:8002"),
CONNECT_CERTIFICATE_GRPC_CRT=(str, None),
REPOSITORY_RESTRICT_ACCESS_NLP_LOGS=(list, []),
REPOSITORY_BLOCK_USER_LOGS=(list, []),
REPOSITORY_KNOWLEDGE_BASE_DESCRIPTION_LIMIT=(int, 450),
REPOSITORY_EXAMPLE_TEXT_WORDS_LIMIT=(int, 200),
ELASTICSEARCH_DSL=(str, "localhost:9200"),
Expand Down Expand Up @@ -263,7 +264,7 @@

envvar_EMAIL_HOST = env.str("EMAIL_HOST")

ADMINS = env.list("ADMINS")
ADMINS = [("Helder", "helder.souza@weni.ai")] # env.list("ADMINS")
EMAIL_SUBJECT_PREFIX = "[bothub] "
DEFAULT_FROM_EMAIL = env.str("DEFAULT_FROM_EMAIL")
SERVER_EMAIL = env.str("SERVER_EMAIL")
Expand Down Expand Up @@ -451,12 +452,12 @@
REPOSITORY_RESTRICT_ACCESS_NLP_LOGS = env.list("REPOSITORY_RESTRICT_ACCESS_NLP_LOGS")

# Limit of characters for the knowledge base description
REPOSITORY_KNOWLEDGE_BASE_DESCRIPTION_LIMIT = env.list(
REPOSITORY_KNOWLEDGE_BASE_DESCRIPTION_LIMIT = env.int(
"REPOSITORY_KNOWLEDGE_BASE_DESCRIPTION_LIMIT", default=450
)

# Limit of words for the example sentence
REPOSITORY_EXAMPLE_TEXT_WORDS_LIMIT = env.list(
REPOSITORY_EXAMPLE_TEXT_WORDS_LIMIT = env.int(
"REPOSITORY_EXAMPLE_TEXT_WORDS_LIMIT", default=200
)

Expand Down Expand Up @@ -582,3 +583,5 @@
ELASTICSEARCH_DSL_SIGNAL_PROCESSOR = ELASTICSEARCH_SIGNAL_PROCESSOR_CLASSES[
env.str("ELASTICSEARCH_SIGNAL_PROCESSOR", default="realtime")
]

REPOSITORY_BLOCK_USER_LOGS = env.list("REPOSITORY_BLOCK_USER_LOGS", default=[])
31 changes: 21 additions & 10 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,24 @@ version: '3.6'

services:
database:
image: postgres
ports:
- 5432:5432
networks:
- default
- bothub
environment:
- POSTGRES_USER=${POSTGRES_USER:-bothub}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-bothub}
- POSTGRES_DB=${POSTGRES_DB:-bothub}
image: postgres
ports:
- 5432:5432
volumes:
- postgres:/var/lib/postgresql/data
networks:
- default
- bothub
environment:
- POSTGRES_USER=${POSTGRES_USER:-bothub}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-bothub}
- POSTGRES_DB=${POSTGRES_DB:-bothub}
deploy:
mode: replicated
replicas: 1
placement:
constraints: [node.role == manager]

bothub:
image: ${DOCKER_IMAGE_NAME:-ilha/bothub}:${TAG:-latest}
build:
Expand Down Expand Up @@ -153,6 +161,9 @@ services:
- default
- bothub

volumes:
postgres:

networks:
bothub:
external: true

0 comments on commit d9f0de7

Please sign in to comment.