merge master

.github/workflows/build-develop.yaml

@@ -1,4 +1,4 @@
-name: Build Develop Image
+name: Build Develop Image Engine
 on:
   push:
     tags:
@@ -9,6 +9,8 @@ jobs:
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
+        with:
+          ref: develop
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -29,4 +31,3 @@ jobs:
           file: ./Dockerfile
           push: true
           tags: bothubit/bothub-engine:${{ steps.vars.outputs.tag }}
-

.github/workflows/build-production.yaml

@@ -0,0 +1,34 @@
+name: Build Production Image
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+        with:
+          ref: master
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Set output
+        id: vars
+        run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
+      - name: Login to DockerHub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: bothubit/bothub-engine:${{ steps.vars.outputs.tag }}
+

.github/workflows/build-staging.yaml

@@ -0,0 +1,34 @@
+name: Build Staging Image Engine
+on:
+  push:
+    tags:
+      - 'staging-v*.*.*'
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+        with:
+          ref: staging
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Set output
+        id: vars
+        run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
+      - name: Login to DockerHub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: bothubit/bothub-engine:${{ steps.vars.outputs.tag }}
+

.gitignore

@@ -118,7 +118,7 @@ staticfiles/
 .idea/
 
 # gRPC Protobuff
-bothub/protos/*.py
+bothub/protos/
 
 #ASDF
 .tool-versions

.travis.yml

@@ -41,10 +41,6 @@ before_script:
   - psql -U postgres -c "CREATE USER bothub WITH PASSWORD 'bothub';"
   - psql -U postgres -c "ALTER ROLE bothub WITH SUPERUSER;"
   - psql -U bothub postgres -c "CREATE DATABASE bothub;"
-  - python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/authentication.proto
-  - python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/organization.proto
-  - python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/repository.proto
-  - python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/project.proto
 
 install:
   - pip install pipenv
@@ -61,6 +57,7 @@ env:
     - ELASTICSEARCH_NUMBER_OF_SHARDS=1
     - ELASTICSEARCH_NUMBER_OF_REPLICAS=1
 script:
+  - make createproto
   - python manage.py migrate
   - python manage.py search_index --rebuild -f
   - python manage.py collectstatic

Dockerfile

@@ -8,6 +8,8 @@ RUN apt-get update \
  && apt-get install --no-install-recommends --no-install-suggests -y gcc bzip2 git curl nginx libpq-dev gettext \
     libgdal-dev python3-cffi python3-gdal vim
 
+RUN apt-get install make
+
 RUN pip install -U pip==21.2.2 setuptools==57.4.0
 RUN pip install pipenv==2021.5.29
 RUN pip install gunicorn==19.9.0
@@ -23,10 +25,7 @@ RUN pipenv install --system
 
 COPY . .
 
-RUN python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/authentication.proto
-RUN python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/organization.proto
-RUN python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/repository.proto
-RUN python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/project.proto
+RUN make createproto
 
 RUN chmod +x ./entrypoint.sh
 ENTRYPOINT [ "./entrypoint.sh" ]

Makefile

@@ -73,11 +73,13 @@ search_index:
 		else pipenv run python manage.py search_index --rebuild -f; fi
 
 createproto:
-	@rm -rf ./bothub/protos/*.py
-	@python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/authentication.proto
-	@python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/organization.proto
-	@python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/repository.proto
-	@python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/project.proto
+
+	@rm -rf ./bothub/protos/
+	@git clone --depth 1 --branch main https://github.com/Ilhasoft/weni-protobuffers ./bothub/protos/
+	@python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/src/weni/protobuf/intelligence/authentication.proto
+	@python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/src/weni/protobuf/intelligence/organization.proto
+	@python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/src/weni/protobuf/intelligence/repository.proto	
+	@python -m grpc_tools.protoc --experimental_allow_proto3_optional --proto_path=./ --python_out=./ --grpc_python_out=./ ./bothub/protos/src/weni/protobuf/connect/project.proto
 
 
 # Utils

bothub/api/grpc/connect_grpc_client.py

@@ -3,7 +3,7 @@
 import grpc
 from django.conf import settings
 
-from bothub.protos import project_pb2_grpc, project_pb2
+from bothub.protos.src.weni.protobuf.connect import project_pb2_grpc, project_pb2
 
 
 class ConnectGRPCClient:

bothub/api/grpc/organization/handlers.py

@@ -1,5 +1,5 @@
 from bothub.api.grpc.organization.services import OrgService
-from bothub.protos import organization_pb2_grpc
+from bothub.protos.src.weni.protobuf.intelligence import organization_pb2_grpc
 
 
 def grpc_handlers(server):

bothub/api/grpc/organization/serializers.py

@@ -4,7 +4,7 @@
 
 from bothub.authentication.models import User
 from bothub.common.models import Organization, OrganizationAuthorization
-from bothub.protos import organization_pb2
+from bothub.protos.src.weni.protobuf.intelligence import organization_pb2
 
 
 class OrgProtoSerializer(proto_serializers.ModelProtoSerializer):

bothub/api/grpc/organization/services.py

@@ -16,7 +16,7 @@
     RepositoryAuthorization,
     Repository,
 )
-from bothub.protos.organization_pb2 import OrgStatistic
+from bothub.protos.src.weni.protobuf.intelligence.organization_pb2 import OrgStatistic
 
 
 class OrgService(

bothub/api/grpc/repository/handlers.py

@@ -1,5 +1,5 @@
 from .services import RepositoryService
-from bothub.protos import repository_pb2_grpc
+from bothub.protos.src.weni.protobuf.intelligence import repository_pb2_grpc
 
 
 def grpc_handlers(server):

bothub/api/grpc/repository/serializers.py

@@ -3,7 +3,7 @@
 
 from bothub.api.v2.repository.serializers import RepositoryCategorySerializer
 from bothub.common.models import Repository
-from bothub.protos import repository_pb2
+from bothub.protos.src.weni.protobuf.intelligence import repository_pb2
 
 
 class RepositoryProtoSerializer(proto_serializers.ModelProtoSerializer):

bothub/api/grpc/repository/tests.py

@@ -8,7 +8,10 @@
     OrganizationAuthorization,
     RepositoryAuthorization,
 )
-from bothub.protos import repository_pb2_grpc, repository_pb2
+from bothub.protos.src.weni.protobuf.intelligence import (
+    repository_pb2_grpc,
+    repository_pb2,
+)
 
 
 class RepositoryServiceTestCase(RPCTransactionTestCase):

bothub/api/grpc/user/handlers.py

@@ -3,7 +3,7 @@
     UserService,
     UserLanguageService,
 )
-from bothub.protos import authentication_pb2_grpc
+from bothub.protos.src.weni.protobuf.intelligence import authentication_pb2_grpc
 
 
 def grpc_handlers(server):

bothub/api/grpc/user/serializers.py

@@ -2,7 +2,7 @@
 from rest_framework import serializers
 
 from bothub.authentication.models import User
-from bothub.protos import authentication_pb2
+from bothub.protos.src.weni.protobuf.intelligence import authentication_pb2
 
 
 class UserProtoSerializer(proto_serializers.ModelProtoSerializer):

bothub/api/grpc/user/tests.py

@@ -3,7 +3,10 @@
 from bothub.api.v2.tests.utils import create_user_and_token
 from bothub.authentication.models import User
 from bothub.common.models import Organization, OrganizationAuthorization
-from bothub.protos import authentication_pb2_grpc, authentication_pb2
+from bothub.protos.src.weni.protobuf.intelligence import (
+    authentication_pb2_grpc,
+    authentication_pb2,
+)
 
 
 class UserServiceTest(RPCTransactionTestCase):

bothub/common/tasks.py

@@ -16,6 +16,7 @@
 from bothub.celery import app
 from bothub.common.models import (
     RepositoryQueueTask,
+    RepositoryReports,
     RepositoryVersion,
     RepositoryVersionLanguage,
     RepositoryExample,
@@ -291,12 +292,13 @@ def delete_nlp_logs():
 def repositories_count_authorizations():
     for repository in Repository.objects.all():
         count = repository.authorizations.filter(
-            user__in=RepositoryNLPLog.objects.filter(
+            user__in=RepositoryReports.objects.filter(
                 repository_version_language__repository_version__repository=repository,
-                from_backend=False,
+                report_date__year=timezone.now().year,
+                report_date__month=timezone.now().month,
             )
             .distinct()
-            .values("user")
+            .values_list("user", flat=True)
         ).count()
         repository.count_authorizations = count
         repository.save(update_fields=["count_authorizations"])