Staging

.github/workflows/build-develop.yaml

@@ -30,4 +30,4 @@ jobs:
           context: .
           file: ./Dockerfile
           push: true
-          tags: bothubit/bothub-engine:${{ steps.vars.outputs.tag }}
+          tags: bothubit/bothub-engine:${{ steps.vars.outputs.tag }}

.gitignore

@@ -74,6 +74,7 @@ target/
 
 # celery beat schedule file
 celerybeat-schedule
+celerybeat-schedule.db
 
 # SageMath parsed files
 *.sage.py
@@ -121,3 +122,6 @@ bothub/protos/
 
 #ASDF
 .tool-versions
+
+#Local Docker Compose file
+dev-docker-compose.yml

.travis.yml

@@ -9,14 +9,20 @@ python:
 services:
   - docker
 
-
 before_install:
   # install the chosen PG version
   - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install postgresql-11 postgresql-client-11
   - sudo -E sed -i -e '/local.*peer/s/postgres/all/' -e 's/peer\|md5/trust/g' /etc/postgresql/*/main/pg_hba.conf
   - sudo -E sed -i 's/port = 5433/port = 5432/' /etc/postgresql/*/main/postgresql.conf
 
-  # give PG some time to finish setup
+  # Install the chosen ES version
+  - curl https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.13.3-amd64.deb -o elasticsearch.deb
+  - sudo dpkg -i --force-confnew elasticsearch.deb
+  - sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch
+  - sudo service elasticsearch restart
+  - until curl --silent -XGET --fail http://localhost:9200; do printf '.'; sleep 1; done
+
+  # give PG and ES some time to finish setup
   - sleep 10
 
   # stop any running postgres versions
@@ -45,9 +51,15 @@ env:
     - SECRET_KEY=SK
     - SUPPORTED_LANGUAGES="en|pt"
     - DEFAULT_DATABASE="postgres://bothub:bothub@localhost:5432/bothub"
+    - ELASTICSEARCH_DSL="localhost:9200"
+    - ELASTICSEARCH_SIGNAL_PROCESSOR="realtime"
+    - ELASTICSEARCH_REPOSITORYNLPLOG_INDEX="repositorynlplog"
+    - ELASTICSEARCH_NUMBER_OF_SHARDS=1
+    - ELASTICSEARCH_NUMBER_OF_REPLICAS=1
 script:
   - make createproto
   - python manage.py migrate
+  - python manage.py search_index --rebuild -f
   - python manage.py collectstatic
   - flake8
   - travis_wait coverage run manage.py test

Pipfile

@@ -4,46 +4,50 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-django = "~=2.2.12"
-djangorestframework = "~=3.9.3"
-django-filter = "~=2.2.0"
-django-cors-headers = "~=3.3.0"
+django = "~=3.2.6"
+djangorestframework = "~=3.12.4"
+django-filter = "~=2.4.0"
+django-cors-headers = "~=3.7.0"
 requests = "~=2.23.0"
 coreapi = "~=2.3.3"
-whitenoise = "~=5.1.0"
+whitenoise = "~=5.3.0"
 pytz = "~=2020.1"
 drf-yasg2 = "~=1.19.4"
 gunicorn = "~=19.9.0"
 gevent = "~=1.4.0"
 packaging = "~=20.4"
-django-environ = "~=0.4.5"
-boto3 = "~=1.13.9"
-sentry-sdk = "~=0.13.2"
-django-celery-beat = "*"
+django-environ-2 = "~=2.1.0"
+boto3 = "~=1.18.13"
+sentry-sdk = "~=1.3.1"
 redis = "*"
-django-celery-results = "~=1.1.2"
-celery = "~=4.4.0"
+celery = "~=5.1.2"
+django-celery-beat = "~=2.2.1"
+django-celery-results = "~=2.2.0"
 python-slugify = "~=4.0.0"
 openpyxl = "~=3.0.3"
-pillow = "~=7.1.2"
-kombu = "~=4.6.7"
-psycopg2-binary = "~=2.7.7"
+pillow = "~=8.3.1"
+kombu = "~=5.1.0"
 matplot = "~=0.1.9"
 django-redis = "~=4.12.1"
-elastic-apm = "~=5.9.0"
-mozilla-django-oidc = "~=1.2.4"
+elastic-apm = "~=6.3.3"
+mozilla-django-oidc = "~=2.0.0"
 djangorestframework-recaptcha = "~=0.2.0"
-djangogrpcframework = "~=0.2"
-grpcio = "~=1.34.0"
-grpcio-tools = "~=1.34.0"
+djangogrpcframework = "~=0.2.1"
+grpcio = "~=1.39.0"
+grpcio-tools = "~=1.39.0"
+django-elasticsearch-dsl= "~=7.2.0"
+django-elasticsearch-dsl-drf= "~=0.22.1"
+elasticsearch= "~=7.14.0"
+elasticsearch-dsl= "~=7.4.0"
+psycopg2-binary = "~=2.9.1"
 
 [dev-packages]
-"flake8" = "*"
+"flake8" = "~=3.9.2"
 requests-mock = "~=1.8.0"
-coverage = "*"
+coverage = "~=5.5"
 ipython = "*"
 autopep8 = "*"
-black = "==19.3b0"
+black = "==21.7b0"
 
 [requires]
 python_version = "3.6"

README.md

@@ -37,6 +37,7 @@ Use ```make``` commands to ```check_environment```, ```install_requirements```,
 | make lint | Show lint warnings and errors
 | make test | Run unit tests and show coverage report
 | make migrate | Update DB shema, apply migrations
+| make search_index | (Re)create and (re)populate the indices in Elasticsearch
 | make start | Start development web server
 | make start_celery | Start celery service
 | make migrations | Create DB migrations files
@@ -155,8 +156,13 @@ You can set environment variables in your OS, write on ```.env``` file or pass v
 | CONNECT_GRPC_SERVER_URL | ```string``` | ```localhost:8002``` | Define grpc connect server url
 | CONNECT_CERTIFICATE_GRPC_CRT | ```string``` | ```None``` | Absolute certificate path for secure grpc communication
 | RECAPTCHA_SECRET_KEY | ```string``` | ```''``` | Token of the recaptcha used in the validation of a user's registration.
-| REPOSITORY_NLP_LOG_LIMIT | ```int``` | ```10000``` | Limit of query size to repository log
+| REPOSITORY_NLP_LOG_LIMIT | ```int``` | ```10000``` | Limit of query size to repository log.
 | REPOSITORY_RESTRICT_ACCESS_NLP_LOGS | ```list``` | ```[]``` | Restricts log access to a particular or multiple intelligences
+| ELASTICSEARCH_DSL | ```string``` | ```es:9200``` | URL Elasticsearch.
+| ELASTICSEARCH_NUMBER_OF_SHARDS | ```int``` | ```1``` | Specify the number of shards for the indexes.
+| ELASTICSEARCH_NUMBER_OF_REPLICAS | ```int``` | ```1``` | Specify the number of replicas for the indexes.
+| ELASTICSEARCH_REPOSITORYNLPLOG_INDEX | ```string``` | ```repositorynlplog``` | Specify the index title for the RepositoryNlpLog document.
+| ELASTICSEARCH_SIGNAL_PROCESSOR | ```string``` | ```celery``` | Specify the signal processor responsible for updating the Elasticsearch data.
 
 
 ## Roadmap

bothub/api/v2/repository/filters.py

@@ -1,16 +1,13 @@
 from django.conf import settings
 from django.core.exceptions import ValidationError as DjangoValidationError
-from django.db.models import Q
 from django.shortcuts import get_object_or_404
 from django.utils.translation import ugettext_lazy as _
 from django_filters import rest_framework as filters
 from rest_framework.exceptions import NotFound
 from rest_framework.exceptions import PermissionDenied
-
 from bothub.authentication.models import RepositoryOwner
 from bothub.common.models import (
     Repository,
-    RepositoryNLPLog,
     RepositoryEntity,
     RepositoryQueueTask,
     RepositoryIntent,
@@ -112,81 +109,44 @@ def filter_repository_uuid(self, queryset, name, value):
             raise NotFound(_("Invalid repository UUID"))
 
 
-class RepositoryNLPLogFilter(filters.FilterSet):
-    class Meta:
-        model = RepositoryNLPLog
-        fields = []
-
-    repository_uuid = filters.CharFilter(
-        field_name="repository_uuid",
-        method="filter_repository_uuid",
-        help_text=_("Repository's UUID"),
-    )
-
-    language = filters.CharFilter(
-        field_name="language",
-        method="filter_language",
-        help_text="Filter by language, default is repository base language",
-    )
-
-    repository_version = filters.CharFilter(
-        field_name="repository_version_language",
-        method="filter_repository_version",
-        help_text=_("Filter for examples with version id."),
-    )
-
-    repository_version_language = filters.CharFilter(
-        field_name="repository_version_language",
-        method="filter_repository_version_language",
-        required=True,
-        help_text=_("Filter for examples with version language id."),
-    )
-
-    intent = filters.CharFilter(
-        field_name="repository_version_language",
-        method="filter_intent",
-        help_text=_("Filter for examples with version name."),
-    )
+class RepositoryNLPLogFilter:
+    def __init__(self, params, user):
+        self.params = params
+        self.user = user
+        self.check = self.check_params()
 
-    confidence = filters.RangeFilter(
-        field_name="repository_version_language", method="filter_confidence"
-    )
+    def check_params(self):
+        validate_param = False
+        for param, value in self.params.items():
+            if value is not None:
+                validate_param = getattr(self, f"check_{param}")(value)
+        return validate_param
 
-    def filter_repository_uuid(self, queryset, name, value):
-        request = self.request
+    def check_repository_uuid(self, value):
         try:
             repository = Repository.objects.get(uuid=value)
-            authorization = repository.get_user_authorization(request.user)
+            authorization = repository.get_user_authorization(self.user)
             if not authorization.can_contribute:
                 raise PermissionDenied()
-            return queryset.filter(
-                repository_version_language__repository_version__repository=repository
-            )
+            return True
         except Repository.DoesNotExist:
             raise NotFound(_("Repository {} does not exist").format(value))
         except DjangoValidationError:
             raise NotFound(_("Invalid repository_uuid"))
 
-    def filter_language(self, queryset, name, value):
-        return queryset.filter(repository_version_language__language=value)
-
-    def filter_repository_version(self, queryset, name, value):
-        request = self.request
+    def check_repository_version(self, value):
         try:
             repository = RepositoryVersion.objects.get(pk=value).repository
-            authorization = repository.get_user_authorization(request.user)
+            authorization = repository.get_user_authorization(self.user)
             if not authorization.can_contribute:
                 raise PermissionDenied()
-            return queryset.filter(
-                repository_version_language__repository_version=value
-            )
+            return True
         except RepositoryVersion.DoesNotExist:
             raise NotFound(_("RepositoryVersion {} does not exist").format(value))
         except DjangoValidationError:
             raise NotFound(_("Invalid repository_version"))
 
-    def filter_repository_version_language(self, queryset, name, value):
-        request = self.request
+    def check_repository_version_language(self, value):
         try:
             repository = RepositoryVersionLanguage.objects.get(
                 pk=value
@@ -198,29 +158,17 @@ def filter_repository_version_language(self, queryset, name, value):
             ):
                 # Restricts log access to a particular or multiple intelligences
                 raise PermissionDenied()
-            authorization = repository.get_user_authorization(request.user)
+            authorization = repository.get_user_authorization(self.user)
             if not authorization.can_contribute:
                 raise PermissionDenied()
-            return queryset.filter(repository_version_language=value)
+            return True
         except RepositoryVersionLanguage.DoesNotExist:
             raise NotFound(
                 _("RepositoryVersionLanguage {} does not exist").format(value)
             )
         except DjangoValidationError:
             raise NotFound(_("Invalid repository_version_language"))
 
-    def filter_intent(self, queryset, name, value):
-        return queryset.filter(
-            repository_nlp_log__intent=value, repository_nlp_log__is_default=True
-        )
-
-    def filter_confidence(self, queryset, name, value):
-        query = queryset.filter(
-            Q(repository_nlp_log__confidence__gte=int(value.start) / 100),
-            Q(repository_nlp_log__confidence__lte=int(value.stop) / 100),
-        )
-        return query
-
 
 class RepositoryEntitiesFilter(filters.FilterSet):
     class Meta:

bothub/api/v2/repository/permissions.py

@@ -1,9 +1,29 @@
 from rest_framework import permissions
 
+from bothub.common.models import RepositoryVersionLanguage
 from .. import READ_METHODS
 from .. import WRITE_METHODS
 
 
+class RepositoryLogPermission(permissions.BasePermission):
+    def has_object_permission(self, request, view):
+        value = self.request.query_params.get("repository_version_language", None)
+        obj = RepositoryVersionLanguage.objects.get(
+            pk=value
+        ).repository_version.repository
+        authorization = obj.get_user_authorization(request.user)
+        if request.method in READ_METHODS and not request.user.is_authenticated:
+            return authorization.can_read
+
+        if request.user.is_authenticated:
+            if request.method in READ_METHODS:
+                return authorization.can_read
+            if request.method in WRITE_METHODS:
+                return authorization.can_write
+            return authorization.is_admin
+        return False
+
+
 class RepositoryPermission(permissions.BasePermission):
     def has_object_permission(self, request, view, obj):
         authorization = obj.get_user_authorization(request.user)