diff --git a/bothub/api/v2/repository/serializers.py b/bothub/api/v2/repository/serializers.py index c0370173..2ecb16e6 100644 --- a/bothub/api/v2/repository/serializers.py +++ b/bothub/api/v2/repository/serializers.py @@ -859,10 +859,11 @@ def get_nlp_server(self, obj): return settings.BOTHUB_NLP_BASE_URL def get_version_default(self, obj): + current_version = obj.repository.current_version() return { - "id": obj.repository.current_version().repository_version.pk, - "repository_version_language_id": obj.repository.current_version().pk, - "name": obj.repository.current_version().repository_version.name, + "id": current_version.repository_version.pk, + "repository_version_language_id": current_version.pk, + "name": current_version.repository_version.name, } def get_repository_score(self, obj): @@ -1672,3 +1673,13 @@ def to_representation(self, instance): if data.get("organization"): data.pop("organization") return data + + +class RepositoryCloneSerializer(serializers.Serializer): + + repository = serializers.PrimaryKeyRelatedField( + queryset=Repository.objects.all(), required=True + ) + owner = serializers.PrimaryKeyRelatedField( + queryset=RepositoryOwner.objects.all(), required=True + ) diff --git a/bothub/api/v2/repository/views.py b/bothub/api/v2/repository/views.py index 386b8ad8..f6d642e9 100644 --- a/bothub/api/v2/repository/views.py +++ b/bothub/api/v2/repository/views.py @@ -119,6 +119,7 @@ WordDistributionSerializer, RemoveRepositoryProject, AddRepositoryProjectSerializer, + RepositoryCloneSerializer, ) from bothub.api.v2.internal.connect_rest_client import ( @@ -133,7 +134,14 @@ class NewRepositoryViewSet( Manager repository (bot). """ - queryset = RepositoryVersion.objects + queryset = ( + RepositoryVersion.objects.all() + .select_related( + "repository", + "repository__owner", + ) + .prefetch_related("repository__categories") + ) lookup_field = "repository__uuid" lookup_fields = ["repository__uuid", "pk"] serializer_class = NewRepositorySerializer @@ -506,6 +514,7 @@ def languagesstatus(self, request, **kwargs): serializer_class=TrainSerializer, ) def train(self, request, **kwargs): + """ Train current update using Bothub NLP service """ @@ -1069,6 +1078,8 @@ def destroy(self, request, *args, **kwargs): class RepositoryTokenByUserViewSet(mixins.ListModelMixin, GenericViewSet): + serializer_class = RepositoryAuthorizationSerializer + def get_queryset(self): user = self.request.user if user.is_anonymous: @@ -1497,3 +1508,20 @@ def get_serializer(self, *args, **kwargs): kwargs["many"] = True return super().get_serializer(*args, **kwargs) + + +class CloneRepositoryViewSet(mixins.CreateModelMixin, GenericViewSet): + queryset = Repository.objects.none() + serializer_class = RepositoryCloneSerializer + permission_classes = (IsAuthenticated,) + + def create(self, request, *args, **kwargs): + serializer = self.serializer_class(data=request.data) + serializer.is_valid(raise_exception=True) + + owner_id = serializer.data.get("owner") + repository_id = serializer.data.get("repository") + repository = Repository.objects.get(pk=repository_id) + + slug, message, http_status = repository.clone_self(owner_id) + return Response(slug if slug else message, status=http_status) diff --git a/bothub/api/v2/routers.py b/bothub/api/v2/routers.py index 73b8db9a..04731378 100644 --- a/bothub/api/v2/routers.py +++ b/bothub/api/v2/routers.py @@ -1,78 +1,81 @@ from rest_framework import routers +from bothub.api.v2.internal.organization.views import InternalOrganizationViewSet +from bothub.api.v2.internal.repository.views import InternalRepositoriesViewSet +from bothub.api.v2.internal.user.views import ( + UserLanguageViewSet, + UserPermissionViewSet, + UserViewSet, +) from bothub.api.v2.versionning.views import RepositoryVersionViewSet -from .account.views import ChangePasswordViewSet -from .account.views import LoginViewSet -from .account.views import MyUserProfileViewSet -from .account.views import RegisterUserViewSet -from .account.views import RequestResetPasswordViewSet -from .account.views import ResetPasswordViewSet -from .account.views import SearchUserViewSet -from .account.views import UserProfileViewSet -from .evaluate.views import EvaluateViewSet -from .evaluate.views import ResultsListViewSet + +from .account.views import ( + ChangePasswordViewSet, + LoginViewSet, + MyUserProfileViewSet, + RegisterUserViewSet, + RequestResetPasswordViewSet, + ResetPasswordViewSet, + SearchUserViewSet, + UserProfileViewSet, +) +from .evaluate.views import EvaluateViewSet, ResultsListViewSet from .examples.views import ExamplesViewSet from .groups.views import RepositoryEntityGroupViewSet from .knowledge_base.views import QAKnowledgeBaseViewSet, QAtextViewSet -from .nlp.views import NLPLangsViewSet, RepositoryQANLPLogsViewSet -from .nlp.views import RepositoryAuthorizationEvaluateViewSet -from .nlp.views import RepositoryAuthorizationInfoViewSet -from .nlp.views import RepositoryAuthorizationParseViewSet from .nlp.views import ( - RepositoryAuthorizationTrainViewSet, - RepositoryNLPLogsViewSet, - RepositoryAuthorizationKnowledgeBaseViewSet, - RepositoryAuthorizationExamplesViewSet, + NLPLangsViewSet, RepositoryAuthorizationAutomaticEvaluateViewSet, + RepositoryAuthorizationEvaluateViewSet, + RepositoryAuthorizationExamplesViewSet, + RepositoryAuthorizationInfoViewSet, + RepositoryAuthorizationKnowledgeBaseViewSet, + RepositoryAuthorizationParseViewSet, RepositoryAuthorizationTrainLanguagesViewSet, + RepositoryAuthorizationTrainViewSet, + RepositoryNLPLogsViewSet, + RepositoryQANLPLogsViewSet, + RepositoryUpdateInterpretersViewSet, ) -from .nlp.views import RepositoryUpdateInterpretersViewSet from .organization.views import ( - OrganizationViewSet, - OrganizationProfileViewSet, OrganizationAuthorizationViewSet, + OrganizationProfileViewSet, + OrganizationViewSet, ) from .repository.views import ( - RepositoriesContributionsViewSet, - RepositoryQANLPLogViewSet, -) -from .repository.views import RepositoriesViewSet -from .repository.views import RepositoryAuthorizationRequestsViewSet -from .repository.views import RepositoryAuthorizationViewSet -from .repository.views import RepositoryTokenByUserViewSet -from .repository.views import RepositoryCategoriesView -from .repository.views import RepositoryExampleViewSet -from .repository.views import RepositoryMigrateViewSet -from .repository.views import ( - RepositoryViewSet, - RepositoryNLPLogViewSet, - RepositoryEntitiesViewSet, + CloneRepositoryViewSet, NewRepositoryViewSet, RasaUploadViewSet, - RepositoryTaskQueueViewSet, + RepositoriesContributionsViewSet, RepositoriesPermissionsViewSet, - RepositoryNLPLogReportsViewSet, + RepositoriesViewSet, + RepositoryAuthorizationRequestsViewSet, + RepositoryAuthorizationViewSet, + RepositoryCategoriesView, + RepositoryEntitiesViewSet, + RepositoryExamplesBulkViewSet, + RepositoryExampleViewSet, RepositoryIntentViewSet, - RepositoryTranslatorInfoViewSet, + RepositoryMigrateViewSet, + RepositoryNLPLogReportsViewSet, + RepositoryNLPLogViewSet, + RepositoryQANLPLogViewSet, + RepositoryTaskQueueViewSet, + RepositoryTokenByUserViewSet, RepositoryTrainInfoViewSet, - RepositoryExamplesBulkViewSet, + RepositoryTranslatorInfoViewSet, + RepositoryViewSet, + RepositoryVotesViewSet, + SearchRepositoriesViewSet, +) +from .translation.views import ( + RepositoryTranslatedExampleViewSet, + RepositoryTranslatedExporterViewSet, ) -from .repository.views import RepositoryVotesViewSet -from .repository.views import SearchRepositoriesViewSet -from .translation.views import RepositoryTranslatedExampleViewSet -from .translation.views import RepositoryTranslatedExporterViewSet from .translator.views import ( - TranslatorExamplesViewSet, RepositoryTranslationTranslatorExampleViewSet, RepositoryTranslatorViewSet, -) - -from bothub.api.v2.internal.repository.views import InternalRepositoriesViewSet -from bothub.api.v2.internal.organization.views import InternalOrganizationViewSet -from bothub.api.v2.internal.user.views import ( - UserPermissionViewSet, - UserViewSet, - UserLanguageViewSet, + TranslatorExamplesViewSet, ) @@ -205,6 +208,8 @@ def get_lookup_regex(self, viewset, lookup_prefix=""): router.register("repository/upload-rasa-file", RasaUploadViewSet) router.register("repository/entity/group", RepositoryEntityGroupViewSet) router.register("repository/repository-migrate", RepositoryMigrateViewSet) +router.register("repository/clone-repository", CloneRepositoryViewSet) + router.register( "repository/nlp/authorization/train", RepositoryAuthorizationTrainViewSet ) diff --git a/bothub/api/v2/tests/test_repository.py b/bothub/api/v2/tests/test_repository.py index 87029a9a..3c3f2897 100644 --- a/bothub/api/v2/tests/test_repository.py +++ b/bothub/api/v2/tests/test_repository.py @@ -9,6 +9,7 @@ from bothub.api.v2.repository.serializers import NewRepositorySerializer from bothub.api.v2.repository.views import ( + CloneRepositoryViewSet, RepositoriesContributionsViewSet, RepositoryEntitiesViewSet, NewRepositoryViewSet, @@ -49,6 +50,29 @@ ) +def get_authorization_header(token=None): + return {"HTTP_AUTHORIZATION": "Token {}".format(token)} if token else {} + + +def request_repository_info(factory, repository, token): + authorization_header = get_authorization_header(token) + + request = factory.get( + "/v2/repository/info/{}/{}/".format( + str(repository.uuid), repository.current_version().repository_version.pk + ), + **authorization_header, + ) + response = NewRepositoryViewSet.as_view({"get": "retrieve"})( + request, + repository__uuid=repository.uuid, + pk=repository.current_version().repository_version.pk, + ) + response.render() + content_data = json.loads(response.content) + return (response, content_data) + + class CreateRepositoryAPITestCase(TestCase): def setUp(self): self.factory = RequestFactory() @@ -66,9 +90,7 @@ def setUp(self): ) def request(self, data, token=None): - authorization_header = ( - {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} if token else {} - ) + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.post( "/v2/repository/repository-details/", data, **authorization_header @@ -120,25 +142,7 @@ def setUp(self): ] def request(self, repository, token=None): - authorization_header = ( - {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} if token else {} - ) - - request = self.factory.get( - "/v2/repository/info/{}/{}/".format( - repository.uuid, repository.current_version().repository_version.pk - ), - **authorization_header, - ) - - response = NewRepositoryViewSet.as_view({"get": "retrieve"})( - request, - repository__uuid=repository.uuid, - pk=repository.current_version().repository_version.pk, - ) - response.render() - content_data = json.loads(response.content) - return (response, content_data) + return request_repository_info(self.factory, repository, token) def test_okay(self): for repository in self.repositories: @@ -169,9 +173,7 @@ def setUp(self): ] def request(self, repository, token=None): - authorization_header = ( - {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} if token else {} - ) + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.get( "/v2/repository/train/info/{}/{}/".format( @@ -220,9 +222,7 @@ def setUp(self): ] def request(self, repository, data={}, token=None): - authorization_header = ( - {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} if token else {} - ) + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.patch( "/v2/repository/repository-details/{}/".format(repository.uuid), @@ -276,25 +276,7 @@ def setUp(self): ] def request(self, repository, token=None): - authorization_header = ( - {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} if token else {} - ) - - request = self.factory.get( - "/v2/repository/info/{}/{}/".format( - repository.uuid, repository.current_version().repository_version.pk - ), - **authorization_header, - ) - - response = NewRepositoryViewSet.as_view({"get": "retrieve"})( - request, - repository__uuid=repository.uuid, - pk=repository.current_version().repository_version.pk, - ) - response.render() - content_data = json.loads(response.content) - return (response, content_data) + return request_repository_info(self.factory, repository, token) def test_authorization_without_user(self): for repository in self.repositories: @@ -336,7 +318,11 @@ def test_authorization_permission_admin_in_organization(self): role=RepositoryAuthorization.ROLE_ADMIN, ) - user, user_token = (self.owner, self.owner_token) if repository.is_private else (self.user, self.user_token) + user, user_token = ( + (self.owner, self.owner_token) + if repository.is_private + else (self.user, self.user_token) + ) response, content_data = self.request(repository, user_token) authorization = content_data.get("authorization") self.assertIsNotNone(authorization) @@ -344,13 +330,17 @@ def test_authorization_permission_admin_in_organization(self): # Assert owner access vs common user access behavior if user is self.owner: - self.assertEqual(authorization.get("level"), OrganizationAuthorization.ROLE_ADMIN) + self.assertEqual( + authorization.get("level"), OrganizationAuthorization.ROLE_ADMIN + ) self.assertTrue(authorization.get("can_contribute")) self.assertTrue(authorization.get("can_write")) self.assertTrue(authorization.get("can_translate")) self.assertTrue(authorization.get("is_admin")) else: # is not owner - self.assertEqual(authorization.get("level"), OrganizationAuthorization.ROLE_USER) + self.assertEqual( + authorization.get("level"), OrganizationAuthorization.ROLE_USER + ) self.assertFalse(authorization.get("can_contribute")) self.assertFalse(authorization.get("can_write")) self.assertFalse(authorization.get("can_translate")) @@ -363,14 +353,18 @@ def test_authorization_permission_admin_in_organization(self): response, content_data = self.request(repository, user_token) authorization = content_data.get("authorization") self.assertIsNotNone(authorization) - self.assertEqual(authorization.get("level"), OrganizationAuthorization.ROLE_USER) + self.assertEqual( + authorization.get("level"), OrganizationAuthorization.ROLE_USER + ) # User should have role==ROLE_USER when they lose their authorization in the repository repository_authorization.delete() response, content_data = self.request(repository, user_token) authorization = content_data.get("authorization") self.assertIsNotNone(authorization) - self.assertEqual(authorization.get("level"), OrganizationAuthorization.ROLE_USER) + self.assertEqual( + authorization.get("level"), OrganizationAuthorization.ROLE_USER + ) class RepositoryAvailableRequestAuthorizationTestCase(TestCase): @@ -388,25 +382,7 @@ def setUp(self): ) def request(self, repository, token=None): - authorization_header = ( - {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} if token else {} - ) - - request = self.factory.get( - "/v2/repository/info/{}/{}/".format( - repository.uuid, repository.current_version().repository_version.pk - ), - **authorization_header, - ) - - response = NewRepositoryViewSet.as_view({"get": "retrieve"})( - request, - repository__uuid=repository.uuid, - pk=repository.current_version().repository_version.pk, - ) - response.render() - content_data = json.loads(response.content) - return (response, content_data) + return request_repository_info(self.factory, repository, token) def test_owner_ever_false(self): response, content_data = self.request(self.repository, self.owner_token) @@ -495,9 +471,7 @@ def setUp(self): ) def request(self, data={}, token=None): - authorization_header = ( - {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} if token else {} - ) + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.get("/v2/repositories/", data, **authorization_header) response = RepositoriesViewSet.as_view({"get": "list"})(request) response.render() @@ -549,9 +523,7 @@ def setUp(self): ) def request(self, data={}, token=None): - authorization_header = ( - {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} if token else {} - ) + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.get("/v2/repositories/", data, **authorization_header) response = RepositoriesViewSet.as_view({"get": "list"})(request) response.render() @@ -630,10 +602,11 @@ def setUp(self): user=self.owner, repository=self.repository ) - def request(self, param, value, token): - authorization_header = {"HTTP_AUTHORIZATION": "Token {}".format(token)} + def request(self, param, value, token=None): + authorization_header = get_authorization_header(token) request = self.factory.get( - "/v2/repository-votes/?{}={}".format(param, value), **authorization_header + "/v2/repository/repository-votes/?{}={}".format(param, value), + **authorization_header, ) response = RepositoryVotesViewSet.as_view({"get": "list"})( request, repository=self.repository.uuid @@ -652,8 +625,8 @@ def test_repository_okay(self): self.assertEqual(response.status_code, status.HTTP_200_OK) def test_private_repository_okay(self): - response, content_data = self.request("repository", self.repository.uuid, "") - self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + response, content_data = self.request("repository", self.repository.uuid) + self.assertEqual(response.status_code, status.HTTP_200_OK) def test_user_okay(self): response, content_data = self.request( @@ -665,8 +638,8 @@ def test_user_okay(self): self.assertEqual(response.status_code, status.HTTP_200_OK) def test_private_user_okay(self): - response, content_data = self.request("user", self.owner.nickname, "") - self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + response, content_data = self.request("user", self.owner.nickname) + self.assertEqual(response.status_code, status.HTTP_200_OK) class NewRepositoryVoteTestCase(TestCase): @@ -684,7 +657,7 @@ def setUp(self): ) def request(self, data, token): - authorization_header = {"HTTP_AUTHORIZATION": "Token {}".format(token)} + authorization_header = get_authorization_header(token) request = self.factory.post( "/v2/repository-votes/", json.dumps(data), @@ -733,7 +706,7 @@ def setUp(self): ) def request(self, token): - authorization_header = {"HTTP_AUTHORIZATION": "Token {}".format(token)} + authorization_header = get_authorization_header(token) request = self.factory.delete( "/v2/repository-votes/{}/".format(str(self.repository.uuid)), **authorization_header, @@ -891,7 +864,7 @@ def setUp(self): self.user_auth.save() def request(self, repository, token): - authorization_header = {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.get( "/v2/repository/authorizations/", {"repository": repository.uuid}, @@ -932,7 +905,7 @@ def setUp(self): ) def request(self, repository, token, user, data): - authorization_header = {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.patch( "/v2/repository/authorizations/{}/{}/".format( repository.uuid, user.nickname @@ -1012,9 +985,7 @@ def setUp(self): admin_autho.save() def request(self, data, token=None): - authorization_header = ( - {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} if token else {} - ) + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.get( "/v2/repository/authorization-requests/", data, **authorization_header ) @@ -1066,9 +1037,7 @@ def setUp(self): ) def request(self, data, token=None): - authorization_header = ( - {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} if token else {} - ) + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.post( "/v2/repository/authorization-requests/", data, **authorization_header ) @@ -1120,9 +1089,7 @@ def setUp(self): admin_autho.save() def request_approve(self, ra, token=None): - authorization_header = ( - {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} if token else {} - ) + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.put( "/v2/repository/authorization-requests/{}/".format(ra.pk), self.factory._encode_data({}, MULTIPART_CONTENT), @@ -1137,9 +1104,7 @@ def request_approve(self, ra, token=None): return (response, content_data) def request_reject(self, ra, token=None): - authorization_header = ( - {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} if token else {} - ) + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.delete( "/v2/repository/authorization-requests/{}/".format(ra.pk), **authorization_header, @@ -1228,7 +1193,7 @@ def setUp(self): ) def request(self, example, token): - authorization_header = {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.get( "/v2/repository/example/{}/".format(example.id), **authorization_header ) @@ -1290,7 +1255,7 @@ def setUp(self): ) def request(self, token): - authorization_header = {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} + authorization_header = get_authorization_header(token.key if token else None) examples = b"""[ { "text": "yes", @@ -1389,7 +1354,7 @@ def setUp(self): ) def request(self, example, token): - authorization_header = {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.delete( "/v2/repository/example/{}/".format(example.id), **authorization_header ) @@ -1462,7 +1427,7 @@ def setUp(self): ) def request(self, example, token, data): - authorization_header = {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.patch( "/v2/repository/example/{}/".format(example.id), json.dumps(data), @@ -1526,7 +1491,7 @@ def setUp(self): ) def request(self, token, data): - authorization_header = {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.post( "/v2/repository/example/", json.dumps(data), @@ -1764,21 +1729,7 @@ def setUp(self): ) def request(self, repository, token): - authorization_header = {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} - request = self.factory.get( - "/v2/repository/info/{}/{}/".format( - str(repository.uuid), repository.current_version().repository_version.pk - ), - **authorization_header, - ) - response = NewRepositoryViewSet.as_view({"get": "retrieve"})( - request, - repository__uuid=repository.uuid, - pk=repository.current_version().repository_version.pk, - ) - response.render() - content_data = json.loads(response.content) - return (response, content_data) + return request_repository_info(self.factory, repository, token) def test_allowed_in_public(self): # owner @@ -1799,9 +1750,7 @@ def test_forbidden_in_private(self): self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) def test_languages_status(self): - authorization_header = { - "HTTP_AUTHORIZATION": "Token {}".format(self.user_token.key) - } + authorization_header = get_authorization_header(self.user_token.key) request = self.factory.get( "/v2/repository/repository-details/{}/languagesstatus/".format( self.repository.uuid @@ -1865,7 +1814,7 @@ def setUp(self): ) def request(self, repository, token, data): - authorization_header = {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.post( "/v2/repository/repository-details/{}/train/".format(str(repository.uuid)), data, @@ -1905,7 +1854,7 @@ def setUp(self): ) def request(self, repository, token, data): - authorization_header = {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.post( "/v2/repository/repository-details/{}/analyze/".format( str(repository.uuid) @@ -1974,9 +1923,7 @@ def setUp(self): current_version.start_training(self.owner) def request(self, data, token=None): - authorization_header = ( - {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} if token else {} - ) + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.get( "/v2/repository/version/", data, **authorization_header ) @@ -2032,7 +1979,7 @@ def setUp(self): self.example_entity.entity.save() def request(self, data, token): - authorization_header = {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.get( "/v2/repository/entities/", data=data, **authorization_header ) @@ -2091,9 +2038,7 @@ def setUp(self): self.repository_version = self.repository.current_version().repository_version def request(self, id, data={}, token=None): - authorization_header = ( - {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} if token else {} - ) + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.patch( "/v2/repository/intent/{}/".format(id), @@ -2168,7 +2113,7 @@ def setUp(self): ] def request(self, token): - authorization_header = {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.post( "/v2/repository/example-bulk/", data=json.dumps(self.data), @@ -2315,7 +2260,7 @@ def setUp(self): ) def request(self, repository, data={}, token=None): - authorization_header = {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.post( "/v2/repository/repository-details/{}/automatic_evaluate/".format( @@ -2362,7 +2307,7 @@ def setUp(self): ) def request(self, repository, data={}, token=None): - authorization_header = {"HTTP_AUTHORIZATION": "Token {}".format(token.key)} + authorization_header = get_authorization_header(token.key if token else None) request = self.factory.get( "/v2/repository/repository-details/{}/check_can_automatic_evaluate/".format( @@ -2411,3 +2356,80 @@ def test_can_evaluate_automatic(self): response, content_data = self.request(self.repository, data, self.owner_token) self.assertTrue(content_data.get("can_run_evaluate_automatic")) self.assertEqual(len(content_data.get("messages")), 0) + + +class RepositoryCloneTestCase(TestCase): + """Test repository_clone API and validate the underlying function, validating all repository fields + and making sure that new fields, such as foreignkeys and many-to-many relationships will cause test failure, + requiring for clone function refactor.""" + + def setUp(self) -> None: + self.factory = RequestFactory() + self.owner, self.owner_token = create_user_and_token("owner") + self.user, self.user_token = create_user_and_token("user") + self.organization = Organization.objects.create(name="Org1", verificated=True) + self.organization.set_user_permission( + self.user, OrganizationAuthorization.ROLE_ADMIN + ) + + # Create categories for repositories + self.category_1 = RepositoryCategory.objects.create(name="Category 1") + self.category_2 = RepositoryCategory.objects.create(name="Category 2") + self.repositories = [ + create_repository_from_mockup(self.owner, **mockup) + for mockup in get_valid_mockups([self.category_1, self.category_2]) + ] + # Create versions and knowledge bases for repositories + for repository in self.repositories: + RepositoryVersion.objects.create( + repository=repository, name="alfa", is_default=False + ) + RepositoryVersion.objects.create( + repository=repository, name="beta", is_default=True + ) + + return super().setUp() + + def request(self, data, token=""): + authorization_header = get_authorization_header(token.key if token else None) + + url = "/v2/repository/clone-repository/" + request = self.factory.post(url, data, **authorization_header) + + response = CloneRepositoryViewSet.as_view({"post": "create"})(request) + response.render() + content_data = json.loads(response.content) + return (response, content_data) + + def test_cannot_clone_private_repository(self): + repository = self.repositories[1] + repository.is_private = True + repository.save() + response, content_data = self.request( + {"repository": repository.pk, "owner": self.organization.pk}, + self.owner_token, + ) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + def test_cannot_call_api_while_unauthenticated(self): + """Validate that only authenticated users can clone repositories with an unauthenticated request""" + repository = self.repositories[1] + response, content_data = self.request( + {"repository": repository.pk, "owner": self.organization.pk}, token=None + ) + self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + + def test_cannot_call_api_with_empty_or_incomplete_body(self): + repository = self.repositories[0] + response, content_data = self.request({}, token=self.user_token) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + response, content_data = self.request( + {"repository": repository.pk}, token=self.user_token + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + response, content_data = self.request( + {"owner": self.organization.pk}, token=self.user_token + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) diff --git a/bothub/api/v2/tests/test_tasks.py b/bothub/api/v2/tests/test_tasks.py new file mode 100755 index 00000000..57422762 --- /dev/null +++ b/bothub/api/v2/tests/test_tasks.py @@ -0,0 +1,140 @@ +from uuid import uuid4 + +from django.test import RequestFactory, TestCase +from django.utils import timezone +from bothub.common.tasks import clone_repository + +from bothub.api.v2.tests.utils import ( + create_repository_from_mockup, + create_user_and_token, + get_valid_mockups, +) +from bothub.common.models import ( + Organization, + OrganizationAuthorization, + QAKnowledgeBase, + Repository, + RepositoryCategory, + RepositoryVersion, +) + + +class RepositoryCloneTestCase(TestCase): + """Test clone_repository task, validating all repository fields""" + + def setUp(self) -> None: + self.factory = RequestFactory() + self.owner, self.owner_token = create_user_and_token("owner") + self.user, self.user_token = create_user_and_token("user") + self.organization = Organization.objects.create(name="Org", verificated=True) + self.organization.set_user_permission( + self.user, OrganizationAuthorization.ROLE_ADMIN + ) + + # Create categories for repositories + self.category_1 = RepositoryCategory.objects.create(name="Category 1") + self.category_2 = RepositoryCategory.objects.create(name="Category 2") + self.repositories = [ + create_repository_from_mockup(self.owner, **mockup) + for mockup in get_valid_mockups([self.category_1, self.category_2]) + ] + # Create versions and knowledge bases for repositories + for repository in self.repositories: + RepositoryVersion.objects.create( + repository=repository, name="beta", is_default=False + ) + RepositoryVersion.objects.create( + repository=repository, name="alfa", is_default=True + ) + + QAKnowledgeBase.objects.create(repository=repository, user=self.user) + QAKnowledgeBase.objects.create(repository=repository, user=self.owner) + + return super().setUp() + + def test_clone_function(self): + + """validate the underlying function result""" + repository = self.repositories[0] + + original_data = { + "pk": repository.pk, + "owner_id": repository.owner_id, + "slug": repository.slug, + "name": repository.name, + "owner": repository.owner, + "is_private": repository.is_private, + "count_authorizations": repository.count_authorizations, + "created_at": repository.created_at, + "categories_ids": repository.categories.all().values_list("id", flat=True), + "versions_ids": repository.versions.all().values_list("id", flat=True), + "knowledge_bases_ids": repository.knowledge_bases.all().values_list( + "id", flat=True + ), + } + clone_created_at = timezone.now() + + # Cloning Now + clone = Repository.objects.create( + owner=self.organization, slug=uuid4().hex[:32] + ) + clone_id = clone_repository(repository.pk, clone.pk, self.organization.pk) + + repository = Repository.objects.get(pk=original_data.get("pk")) + + # Ensure original Repository fields were not altered. + self.assertEqual(original_data.get("owner_id"), getattr(repository, "owner_id")) + self.assertEqual(original_data.get("slug"), getattr(repository, "slug")) + self.assertEqual(original_data.get("name"), getattr(repository, "name")) + self.assertEqual( + original_data.get("is_private"), getattr(repository, "is_private") + ) + self.assertEqual( + original_data.get("count_authorizations"), + getattr(repository, "count_authorizations"), + ) + self.assertEqual( + original_data.get("created_at"), getattr(repository, "created_at") + ) + repository_categories = repository.categories.all().values_list("id", flat=True) + repository_versions = repository.versions.all().values_list("id", flat=True) + repository_knowledge_bases = repository.knowledge_bases.all().values_list( + "id", flat=True + ) + self.assertEqual( + set(original_data.get("categories_ids")), set(repository_categories) + ) + self.assertEqual( + set(original_data.get("versions_ids")), set(repository_versions) + ) + self.assertEqual( + set(original_data.get("knowledge_bases_ids")), + set(repository_knowledge_bases), + ) + + self.assertEqual(clone_id, clone.pk) + clone.refresh_from_db() + + # Ensure clone fields were inherited + self.assertEqual(clone.repository_type, repository.repository_type) + self.assertEqual(clone.algorithm, repository.algorithm) + self.assertEqual(clone.language, repository.language) + self.assertEqual(clone.use_competing_intents, repository.use_competing_intents) + self.assertEqual(clone.use_name_entities, repository.use_name_entities) + self.assertEqual(clone.use_analyze_char, repository.use_analyze_char) + self.assertEqual(clone.description, repository.description) + + # Ensure clone fields were not inherited + self.assertEqual(clone.owner_id, self.organization.pk) + self.assertEqual(clone.is_private, True) + self.assertEqual(clone.count_authorizations, 0) + self.assertGreaterEqual(clone.created_at, clone_created_at) + + clone_categories = clone.categories.all().values_list("id", flat=True) + self.assertNotEqual(len(clone_categories), 0) + self.assertEqual(len(clone_categories), len(repository_categories)) + self.assertEqual(set(clone_categories), set(repository_categories)) + + clone_knowledge_bases = clone.knowledge_bases.all().values_list("id", flat=True) + self.assertNotEqual(len(clone_knowledge_bases), 0) + self.assertEqual(len(clone_knowledge_bases), len(repository_knowledge_bases)) diff --git a/bothub/api/v2/versionning/serializers.py b/bothub/api/v2/versionning/serializers.py index 1ef451cb..f33109f1 100644 --- a/bothub/api/v2/versionning/serializers.py +++ b/bothub/api/v2/versionning/serializers.py @@ -69,7 +69,7 @@ def create(self, validated_data): # pragma: no cover ) instance.save() answer_task = celery_app.send_task( - "clone_version", args=[instance.pk, id_clone, repository.pk] + "clone_version", args=[repository.pk, instance.pk, id_clone] ) answer_task.wait() return instance diff --git a/bothub/common/models.py b/bothub/common/models.py index 3c8574f3..d3dc2d0c 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -1,3 +1,4 @@ +from typing import Tuple import uuid from functools import reduce @@ -914,6 +915,38 @@ def get_absolute_url(self): settings.BOTHUB_WEBAPP_BASE_URL, self.owner.nickname, self.slug ) + def clone_self(self, new_owner_id) -> Tuple[bool, str, int]: + """Clone the current repository and transfer the clone version to a new owner. + Returns a Tuple[slug:str, message:str, http_status:int]. + (The "slug" field refers to the repository clone's slug field). + """ + from celery import group + from bothub.common.tasks import clone_repository, clone_version + from bothub.utils import unique_slug_generator + + if self.is_private: + return ( + None, + "You cannot clone a private repository", + status.HTTP_403_FORBIDDEN, + ) + + try: + RepositoryOwner.objects.get(id=new_owner_id) + except RepositoryOwner.DoesNotExist: + return None, "User does not exist", status.HTTP_404_NOT_FOUND + + slug = unique_slug_generator({"name": self.name}) + clone = Repository.objects.create(owner_id=new_owner_id, slug=slug) + + # Queue clone_repository and clone_version tasks + group_tasks = group( + clone_repository.s(self.pk, clone.pk, new_owner_id), + *[clone_version.s(clone.pk, version.pk) for version in self.versions.all()], + ) + group_tasks() + return clone.pk, "Queued for cloning", status.HTTP_200_OK + class RepositoryVersion(models.Model): class Meta: diff --git a/bothub/common/tasks.py b/bothub/common/tasks.py index 1660e7f0..87c3eaa5 100644 --- a/bothub/common/tasks.py +++ b/bothub/common/tasks.py @@ -95,16 +95,21 @@ def trainings_check_task(): @app.task(name="clone_version") -def clone_version(instance_id, id_clone, repository, *args, **kwargs): - clone = RepositoryVersion.objects.get(pk=id_clone, repository=repository) +def clone_version( + repository_id: str, instance_id: int, clone_id: int = None, *args, **kwargs +): + if clone_id: + clone = RepositoryVersion.objects.get(pk=clone_id, repository_id=repository_id) + else: + clone = RepositoryVersion.objects.create(repository_id=repository_id) instance = RepositoryVersion.objects.get(pk=instance_id) - bulk_versionlanguages = [ + bulk_version_languages = [ RepositoryVersionLanguage(**version, pk=None, repository_version=instance) for version in clone.version_languages.values() ] RepositoryVersionLanguage.objects.bulk_create( - bulk_versionlanguages, ignore_conflicts=True + bulk_version_languages, ignore_conflicts=True ) for version in clone.version_languages: @@ -129,11 +134,11 @@ def clone_version(instance_id, id_clone, repository, *args, **kwargs): last_update=example.last_update, ) - example_entites = RepositoryExampleEntity.objects.filter( + example_entities = RepositoryExampleEntity.objects.filter( repository_example=example ) - for example_entity in example_entites: + for example_entity in example_entities: if example_entity.entity.group: group, created_group = RepositoryEntityGroup.objects.get_or_create( repository_version=instance, @@ -246,6 +251,61 @@ def clone_version(instance_id, id_clone, repository, *args, **kwargs): return True +@app.task() +def clone_repository( + source_repository_id: str, clone_repository_id: str, new_owner_id: int +) -> Repository: + """ + Clone a Repository Instance ans it's related fields. + Returns a Repository instance or None + """ + + source_repository = Repository.objects.get(pk=source_repository_id) + clone_repository = Repository.objects.get(pk=clone_repository_id) + + # region 1. clone Repository and direct fields + + # copy fields from source repository to clone repository + exclude_fields = ("id", "pk", "uuid", "slug") + for field in Repository._meta.fields: + if field.name in exclude_fields or field.primary_key: + continue + setattr( + clone_repository, + field.name, + getattr(source_repository, field.name), + ) + + # Keep full name if the field's "max_length" allows it, else crop it + size = Repository.name.field.max_length + name = f"Clone - {source_repository.name}" + if len(name) > size: + name = name[: size - 3] + "..." + clone_repository.name = name + + # Update necessary fields + clone_repository.owner_id = new_owner_id + clone_repository.is_private = True + clone_repository.count_authorizations = 0 + clone_repository.created_at = timezone.now() + + clone_repository.save() + # endregion + + # region 2. ForeignKeys and ManyToManyFields + clone_repository.categories.set(source_repository.categories.all()) + # endregion + + # region 3. reverse ForeignKeys and ManyToManyFields (the ones which reference the Repository) + for knowledge_base in source_repository.knowledge_bases.all(): + knowledge_base.pk = None + knowledge_base.repository = clone_repository + knowledge_base.save() + # endregion + + return clone_repository.pk + + @app.task() def delete_nlp_logs(): BATCH_SIZE = 5000 diff --git a/bothub/settings.py b/bothub/settings.py index 4acaaeea..ff653783 100644 --- a/bothub/settings.py +++ b/bothub/settings.py @@ -277,7 +277,7 @@ envvar_EMAIL_HOST = env.str("EMAIL_HOST") -ADMINS = [("Helder", "helder.souza@weni.ai")] # env.list("ADMINS") +ADMINS = env.list("ADMINS") EMAIL_SUBJECT_PREFIX = "[bothub] " DEFAULT_FROM_EMAIL = env.str("DEFAULT_FROM_EMAIL") SERVER_EMAIL = env.str("SERVER_EMAIL") diff --git a/docker-compose.yml b/docker-compose.yml index 56756893..97309ccb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,7 @@ version: '3.6' services: database: image: postgres + container_name: bothub-db ports: - 5432:5432 volumes: @@ -27,6 +28,7 @@ services: bothub: image: ${DOCKER_IMAGE_NAME:-ilha/bothub}:${TAG:-latest} + container_name: bothub-backend build: context: . dockerfile: Dockerfile @@ -79,6 +81,7 @@ services: build: context: . dockerfile: Dockerfile + container_name: bothub-celery depends_on: - bothub - bothub-engine-celery-redis @@ -124,11 +127,13 @@ services: bothub-engine-celery-redis: image: redis + container_name: bothub-redis ports: - 6379:6379 es: image: elasticsearch:7.14.1 + container_name: bothub-es environment: - discovery.type=single-node - bootstrap.memory_lock=true @@ -155,6 +160,7 @@ services: kibana: image: docker.elastic.co/kibana/kibana:7.14.1 + container_name: bothub-kibana environment: SERVER_NAME: kibana ELASTICSEARCH_HOSTS: http://es:9200