diff --git a/docs/webhooks.rst b/docs/webhooks.rst index 4e5bc241946..cfeb604591d 100644 --- a/docs/webhooks.rst +++ b/docs/webhooks.rst @@ -36,7 +36,13 @@ After you have added the integration, you'll see a link to information about the As an example, the URL pattern looks like this: *https://readthedocs.org/api/v2/webhook///*. -Use this URL when setting up a new webhook with your provider -- these steps vary depending on the provider: +Use this URL when setting up a new webhook with your provider -- these steps vary depending on the provider. + +.. note:: + + If your account is connected to the provider, + we'll try to setup the webhook automatically. + If something fails, you can still setup the webhook manually. .. _webhook-integration-github: @@ -161,8 +167,6 @@ we create a secret for every integration that offers a way to verify that a webh Currently, `GitHub `__ and `GitLab `__ offer a way to check this. -When :ref:`resyncing the webhook `, the secret is changed too. - Troubleshooting --------------- diff --git a/readthedocs/integrations/models.py b/readthedocs/integrations/models.py index 55059853e55..bef588535ec 100644 --- a/readthedocs/integrations/models.py +++ b/readthedocs/integrations/models.py @@ -274,6 +274,10 @@ def recreate_secret(self): self.secret = get_secret() self.save(update_fields=['secret']) + def remove_secret(self): + self.secret = None + self.save(update_fields=['secret']) + def __str__(self): return ( _('{0} for {1}') diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index 98a12b58f3e..e093ae24408 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -226,6 +226,19 @@ def get_paginated_results(self, response): """ raise NotImplementedError + def get_provider_data(self, project, integration): + """ + Gets provider data from Git Providers Webhooks API. + + :param project: project + :type project: Project + :param integration: Integration for the project + :type integration: Integration + :returns: Dictionary containing provider data from the API or None + :rtype: dict + """ + raise NotImplementedError + def setup_webhook(self, project, integration=None): """ Setup webhook for project. diff --git a/readthedocs/oauth/services/bitbucket.py b/readthedocs/oauth/services/bitbucket.py index 5af74cf2a12..144b5e4508c 100644 --- a/readthedocs/oauth/services/bitbucket.py +++ b/readthedocs/oauth/services/bitbucket.py @@ -206,6 +206,71 @@ def get_webhook_data(self, project, integration): 'events': ['repo:push'], }) + def get_provider_data(self, project, integration): + """ + Gets provider data from BitBucket Webhooks API. + + :param project: project + :type project: Project + :param integration: Integration for the project + :type integration: Integration + :returns: Dictionary containing provider data from the API or None + :rtype: dict + """ + + if integration.provider_data: + return integration.provider_data + + session = self.get_session() + owner, repo = build_utils.get_bitbucket_username_repo(url=project.repo) + + rtd_webhook_url = 'https://{domain}{path}'.format( + domain=settings.PRODUCTION_DOMAIN, + path=reverse( + 'api_webhook', + kwargs={ + 'project_slug': project.slug, + 'integration_pk': integration.pk, + }, + ), + ) + + try: + resp = session.get( + ( + 'https://api.bitbucket.org/2.0/repositories/{owner}/{repo}/hooks' + .format(owner=owner, repo=repo) + ), + ) + + if resp.status_code == 200: + recv_data = resp.json() + + for webhook_data in recv_data["values"]: + if webhook_data["url"] == rtd_webhook_url: + integration.provider_data = webhook_data + integration.save() + + log.info( + 'Bitbucket integration updated with provider data for project: %s', + project, + ) + break + else: + log.info( + 'Bitbucket project does not exist or user does not have ' + 'permissions: project=%s', + project, + ) + + except Exception: + log.exception( + 'Bitbucket webhook Listing failed for project: %s', + project, + ) + + return integration.provider_data + def setup_webhook(self, project, integration=None): """ Set up Bitbucket project webhook for project. @@ -219,6 +284,7 @@ def setup_webhook(self, project, integration=None): """ session = self.get_session() owner, repo = build_utils.get_bitbucket_username_repo(url=project.repo) + if not integration: integration, _ = Integration.objects.get_or_create( project=project, @@ -251,7 +317,6 @@ def setup_webhook(self, project, integration=None): 'permissions: project=%s', project, ) - return (False, resp) # Catch exceptions with request or deserializing JSON except (RequestException, ValueError): @@ -271,7 +336,8 @@ def setup_webhook(self, project, integration=None): ) except ValueError: pass - return (False, resp) + + return (False, resp) def update_webhook(self, project, integration): """ @@ -284,17 +350,25 @@ def update_webhook(self, project, integration): :returns: boolean based on webhook set up success, and requests Response object :rtype: (Bool, Response) """ + provider_data = self.get_provider_data(project, integration) + + # Handle the case where we don't have a proper provider_data set + # This happens with a user-managed webhook previously + if not provider_data: + return self.setup_webhook(project, integration) + session = self.get_session() data = self.get_webhook_data(project, integration) resp = None try: # Expect to throw KeyError here if provider_data is invalid - url = integration.provider_data['links']['self']['href'] + url = provider_data['links']['self']['href'] resp = session.put( url, data=data, headers={'content-type': 'application/json'}, ) + if resp.status_code == 200: recv_data = resp.json() integration.provider_data = recv_data @@ -308,7 +382,7 @@ def update_webhook(self, project, integration): # Bitbucket returns 404 when the webhook doesn't exist. In this # case, we call ``setup_webhook`` to re-configure it from scratch if resp.status_code == 404: - return self.setup_webhook(project) + return self.setup_webhook(project, integration) # Catch exceptions with request or deserializing JSON except (KeyError, RequestException, TypeError, ValueError): @@ -316,7 +390,6 @@ def update_webhook(self, project, integration): 'Bitbucket webhook update failed for project: %s', project, ) - return (False, resp) else: log.error( 'Bitbucket webhook update failed for project: %s', @@ -331,4 +404,5 @@ def update_webhook(self, project, integration): 'Bitbucket webhook update failure response: %s', debug_data, ) - return (False, resp) + + return (False, resp) diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index c5b1f2de8ce..406c413a421 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -190,6 +190,71 @@ def get_webhook_data(self, project, integration): 'events': ['push', 'pull_request', 'create', 'delete'], }) + def get_provider_data(self, project, integration): + """ + Gets provider data from GitHub Webhooks API. + + :param project: project + :type project: Project + :param integration: Integration for the project + :type integration: Integration + :returns: Dictionary containing provider data from the API or None + :rtype: dict + """ + + if integration.provider_data: + return integration.provider_data + + session = self.get_session() + owner, repo = build_utils.get_github_username_repo(url=project.repo) + + rtd_webhook_url = 'https://{domain}{path}'.format( + domain=settings.PRODUCTION_DOMAIN, + path=reverse( + 'api_webhook', + kwargs={ + 'project_slug': project.slug, + 'integration_pk': integration.pk, + }, + ) + ) + + try: + resp = session.get( + ( + 'https://api.github.com/repos/{owner}/{repo}/hooks' + .format(owner=owner, repo=repo) + ), + ) + + if resp.status_code == 200: + recv_data = resp.json() + + for webhook_data in recv_data: + if webhook_data["config"]["url"] == rtd_webhook_url: + integration.provider_data = webhook_data + integration.save() + + log.info( + 'GitHub integration updated with provider data for project: %s', + project, + ) + break + else: + log.info( + 'GitHub project does not exist or user does not have ' + 'permissions: project=%s', + project, + ) + + except Exception: + log.exception( + 'GitHub webhook Listing failed for project: %s', + project, + ) + + return integration.provider_data + def setup_webhook(self, project, integration=None): """ Set up GitHub project webhook for project. @@ -203,13 +268,16 @@ def setup_webhook(self, project, integration=None): """ session = self.get_session() owner, repo = build_utils.get_github_username_repo(url=project.repo) - if integration: - integration.recreate_secret() - else: + + if not integration: integration, _ = Integration.objects.get_or_create( project=project, integration_type=Integration.GITHUB_WEBHOOK, ) + + if not integration.secret: + integration.recreate_secret() + data = self.get_webhook_data(project, integration) resp = None try: @@ -221,6 +289,7 @@ def setup_webhook(self, project, integration=None): data=data, headers={'content-type': 'application/json'}, ) + # GitHub will return 200 if already synced if resp.status_code in [200, 201]: recv_data = resp.json() @@ -238,10 +307,9 @@ def setup_webhook(self, project, integration=None): 'permissions: project=%s', project, ) - # Set the secret to None so that the integration can be used manually. - integration.secret = None - integration.save() - return (False, resp) + + # All other status codes will flow to the `else` clause below + # Catch exceptions with request or deserializing JSON except (RequestException, ValueError): log.exception( @@ -263,7 +331,10 @@ def setup_webhook(self, project, integration=None): 'GitHub webhook creation failure response: %s', debug_data, ) - return (False, resp) + + # Always remove the secret and return False if we don't return True above + integration.remove_secret() + return (False, resp) def update_webhook(self, project, integration): """ @@ -277,23 +348,34 @@ def update_webhook(self, project, integration): :rtype: (Bool, Response) """ session = self.get_session() - integration.recreate_secret() + if not integration.secret: + integration.recreate_secret() data = self.get_webhook_data(project, integration) resp = None + + provider_data = self.get_provider_data(project, integration) + + # Handle the case where we don't have a proper provider_data set + # This happens with a user-managed webhook previously + if not provider_data: + return self.setup_webhook(project, integration) + try: - url = integration.provider_data.get('url') + url = provider_data.get('url') + resp = session.patch( url, data=data, headers={'content-type': 'application/json'}, ) + # GitHub will return 200 if already synced if resp.status_code in [200, 201]: recv_data = resp.json() integration.provider_data = recv_data integration.save() log.info( - 'GitHub webhook creation successful for project: %s', + 'GitHub webhook update successful for project: %s', project, ) return (True, resp) @@ -301,7 +383,7 @@ def update_webhook(self, project, integration): # GitHub returns 404 when the webhook doesn't exist. In this case, # we call ``setup_webhook`` to re-configure it from scratch if resp.status_code == 404: - return self.setup_webhook(project) + return self.setup_webhook(project, integration) # Catch exceptions with request or deserializing JSON except (AttributeError, RequestException, ValueError): @@ -309,7 +391,6 @@ def update_webhook(self, project, integration): 'GitHub webhook update failed for project: %s', project, ) - return (False, resp) else: log.error( 'GitHub webhook update failed for project: %s', @@ -320,10 +401,12 @@ def update_webhook(self, project, integration): except ValueError: debug_data = resp.content log.debug( - 'GitHub webhook creation failure response: %s', + 'GitHub webhook update failure response: %s', debug_data, ) - return (False, resp) + + integration.remove_secret() + return (False, resp) def send_build_status(self, build, commit, state): """ diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index c22202375f2..04a841786a0 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -14,7 +14,7 @@ RTD_BUILD_STATUS_API_NAME, SELECT_BUILD_STATUS, ) -from readthedocs.builds.utils import get_gitlab_username_repo +from readthedocs.builds import utils as build_utils from readthedocs.integrations.models import Integration from readthedocs.projects.models import Project @@ -52,11 +52,11 @@ def _get_repo_id(self, project): # https://docs.gitlab.com/ce/api/README.html#namespaced-path-encoding try: repo_id = json.loads(project.remote_repository.json).get('id') - except Project.remote_repository.RelatedObjectDoesNotExist: + except Exception: # Handle "Manual Import" when there is no RemoteRepository # associated with the project. It only works with gitlab.com at the # moment (doesn't support custom gitlab installations) - username, repo = get_gitlab_username_repo(project.repo) + username, repo = build_utils.get_gitlab_username_repo(project.repo) if (username, repo) == (None, None): return None @@ -267,6 +267,75 @@ def get_webhook_data(self, repo_id, project, integration): 'wiki_events': False, }) + def get_provider_data(self, project, integration): + """ + Gets provider data from GitLab Webhooks API. + + :param project: project + :type project: Project + :param integration: Integration for the project + :type integration: Integration + :returns: Dictionary containing provider data from the API or None + :rtype: dict + """ + + if integration.provider_data: + return integration.provider_data + + repo_id = self._get_repo_id(project) + + if repo_id is None: + return None + + session = self.get_session() + + rtd_webhook_url = 'https://{domain}{path}'.format( + domain=settings.PRODUCTION_DOMAIN, + path=reverse( + 'api_webhook', + kwargs={ + 'project_slug': project.slug, + 'integration_pk': integration.pk, + }, + ) + ) + + try: + resp = session.get( + '{url}/api/v4/projects/{repo_id}/hooks'.format( + url=self.adapter.provider_base_url, + repo_id=repo_id, + ), + ) + + if resp.status_code == 200: + recv_data = resp.json() + + for webhook_data in recv_data: + if webhook_data["url"] == rtd_webhook_url: + integration.provider_data = webhook_data + integration.save() + + log.info( + 'GitLab integration updated with provider data for project: %s', + project, + ) + break + else: + log.info( + 'GitLab project does not exist or user does not have ' + 'permissions: project=%s', + project, + ) + + except Exception: + log.exception( + 'GitLab webhook Listing failed for project: %s', + project, + ) + + return integration.provider_data + def setup_webhook(self, project, integration=None): """ Set up GitLab project webhook for project. @@ -278,23 +347,26 @@ def setup_webhook(self, project, integration=None): :returns: boolean based on webhook set up success :rtype: bool """ - if integration: - integration.recreate_secret() - else: + resp = None + + if not integration: integration, _ = Integration.objects.get_or_create( project=project, integration_type=Integration.GITLAB_WEBHOOK, ) + + if not integration.secret: + integration.recreate_secret() + repo_id = self._get_repo_id(project) + if repo_id is None: # Set the secret to None so that the integration can be used manually. - integration.secret = None - integration.save() - return (False, None) + integration.remove_secret() + return (False, resp) data = self.get_webhook_data(repo_id, project, integration) session = self.get_session() - resp = None try: resp = session.post( '{url}/api/v4/projects/{repo_id}/hooks'.format( @@ -304,6 +376,7 @@ def setup_webhook(self, project, integration=None): data=data, headers={'content-type': 'application/json'}, ) + if resp.status_code == 201: integration.provider_data = resp.json() integration.save() @@ -319,23 +392,21 @@ def setup_webhook(self, project, integration=None): 'permissions: project=%s', project, ) - # Set the secret to None so that the integration can be used manually. - integration.secret = None - integration.save() - return (False, resp) except (RequestException, ValueError): log.exception( 'GitLab webhook creation failed for project: %s', project, ) - return (False, resp) else: log.error( 'GitLab webhook creation failed for project: %s', project, ) - return (False, resp) + + # Always remove secret and return False if we don't return True above + integration.remove_secret() + return (False, resp) def update_webhook(self, project, integration): """ @@ -352,17 +423,27 @@ def update_webhook(self, project, integration): :rtype: (Bool, Response) """ - session = self.get_session() + provider_data = self.get_provider_data(project, integration) + + # Handle the case where we don't have a proper provider_data set + # This happens with a user-managed webhook previously + if not provider_data: + return self.setup_webhook(project, integration) + resp = None + session = self.get_session() repo_id = self._get_repo_id(project) + if repo_id is None: - return (False, None) + return (False, resp) + + if not integration.secret: + integration.recreate_secret() - integration.recreate_secret() data = self.get_webhook_data(repo_id, project, integration) - hook_id = integration.provider_data.get('id') - resp = None + try: + hook_id = provider_data.get('id') resp = session.put( '{url}/api/v4/projects/{repo_id}/hooks/{hook_id}'.format( url=self.adapter.provider_base_url, @@ -372,6 +453,7 @@ def update_webhook(self, project, integration): data=data, headers={'content-type': 'application/json'}, ) + if resp.status_code == 200: recv_data = resp.json() integration.provider_data = recv_data @@ -385,10 +467,10 @@ def update_webhook(self, project, integration): # GitLab returns 404 when the webhook doesn't exist. In this case, # we call ``setup_webhook`` to re-configure it from scratch if resp.status_code == 404: - return self.setup_webhook(project) + return self.setup_webhook(project, integration) # Catch exceptions with request or deserializing JSON - except (RequestException, ValueError): + except (AttributeError, RequestException, ValueError): log.exception( 'GitLab webhook update failed for project: %s', project, @@ -403,7 +485,9 @@ def update_webhook(self, project, integration): except ValueError: debug_data = resp.content log.debug('GitLab webhook update failure response: %s', debug_data) - return (False, resp) + + integration.remove_secret() + return (False, resp) def send_build_status(self, build, commit, state): """ @@ -418,13 +502,14 @@ def send_build_status(self, build, commit, state): :returns: boolean based on commit status creation was successful or not. :rtype: Bool """ + resp = None session = self.get_session() project = build.project repo_id = self._get_repo_id(project) if repo_id is None: - return (False, None) + return (False, resp) # select the correct state and description. gitlab_build_state = SELECT_BUILD_STATUS[state]['gitlab'] @@ -443,8 +528,6 @@ def send_build_status(self, build, commit, state): } url = self.adapter.provider_base_url - resp = None - try: resp = session.post( f'{url}/api/v4/projects/{repo_id}/statuses/{commit}', diff --git a/readthedocs/oauth/utils.py b/readthedocs/oauth/utils.py index 2370918b712..babb766488b 100644 --- a/readthedocs/oauth/utils.py +++ b/readthedocs/oauth/utils.py @@ -49,6 +49,7 @@ def update_webhook(project, integration, request=None): project.has_valid_webhook = True project.save() return True + messages.error( request, _( diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index a2e4f07f9f7..e73382367dd 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -4,11 +4,17 @@ from django.contrib.auth.models import User from django.test import TestCase from django.test.utils import override_settings +from django.urls import reverse from django_dynamic_fixture import get from readthedocs.builds.constants import EXTERNAL, BUILD_STATUS_SUCCESS from readthedocs.builds.models import Version, Build +from readthedocs.integrations.models import ( + GitHubWebhook, + GitLabWebhook, + BitbucketWebhook +) from readthedocs.oauth.models import RemoteOrganization, RemoteRepository from readthedocs.oauth.services import ( BitbucketService, @@ -34,6 +40,21 @@ def setUp(self): self.external_build = get( Build, project=self.project, version=self.external_version ) + self.integration = get( + GitHubWebhook, + project=self.project, + provider_data={ + 'url': 'https://github.com/' + } + ) + self.provider_data = [ + { + "config": { + "url": "https://example.com/webhook" + }, + "url": "https://api.github.com/repos/test/Hello-World/hooks/12345678", + } + ] def test_make_project_pass(self): repo_json = { @@ -216,6 +237,205 @@ def test_make_private_project(self): repo = self.service.create_repository(repo_json, organization=self.org) self.assertIsNotNone(repo) + @mock.patch('readthedocs.oauth.services.github.log') + @mock.patch('readthedocs.oauth.services.github.GitHubService.get_session') + def test_setup_webhook_successful(self, session, mock_logger): + session().post.return_value.status_code = 201 + session().post.return_value.json.return_value = {} + success, _ = self.service.setup_webhook( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertTrue(success) + self.assertIsNotNone(self.integration.secret) + mock_logger.info.assert_called_with( + "GitHub webhook creation successful for project: %s", + self.project, + ) + + @mock.patch('readthedocs.oauth.services.github.log') + @mock.patch('readthedocs.oauth.services.github.GitHubService.get_session') + def test_setup_webhook_404_error(self, session, mock_logger): + session().post.return_value.status_code = 404 + success, _ = self.service.setup_webhook( + self.project, + self.integration + ) + self.integration.refresh_from_db() + + self.assertFalse(success) + self.assertIsNone(self.integration.secret) + mock_logger.info.assert_called_with( + 'GitHub project does not exist or user does not have ' + 'permissions: project=%s', + self.project, + ) + + @mock.patch('readthedocs.oauth.services.github.log') + @mock.patch('readthedocs.oauth.services.github.GitHubService.get_session') + def test_setup_webhook_value_error(self, session, mock_logger): + session().post.side_effect = ValueError + success = self.service.setup_webhook( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertIsNone(self.integration.secret) + mock_logger.exception.assert_called_with( + 'GitHub webhook creation failed for project: %s', + self.project, + ) + + @mock.patch('readthedocs.oauth.services.github.log') + @mock.patch('readthedocs.oauth.services.github.GitHubService.get_session') + def test_update_webhook_successful(self, session, mock_logger): + session().patch.return_value.status_code = 201 + session().patch.return_value.json.return_value = {} + success, _ = self.service.update_webhook( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertTrue(success) + self.assertIsNotNone(self.integration.secret) + mock_logger.info.assert_called_with( + "GitHub webhook update successful for project: %s", + self.project, + ) + + @mock.patch('readthedocs.oauth.services.github.GitHubService.get_session') + @mock.patch('readthedocs.oauth.services.github.GitHubService.setup_webhook') + def test_update_webhook_404_error(self, setup_webhook, session): + session().patch.return_value.status_code = 404 + self.service.update_webhook( + self.project, + self.integration + ) + + setup_webhook.assert_called_once_with( + self.project, + self.integration + ) + + @mock.patch('readthedocs.oauth.services.github.GitHubService.get_session') + @mock.patch('readthedocs.oauth.services.github.GitHubService.setup_webhook') + def test_update_webhook_no_provider_data(self, setup_webhook, session): + self.integration.provider_data = None + self.integration.save() + + session().patch.side_effect = AttributeError + self.service.update_webhook( + self.project, + self.integration + ) + + setup_webhook.assert_called_once_with( + self.project, + self.integration + ) + + @mock.patch('readthedocs.oauth.services.github.log') + @mock.patch('readthedocs.oauth.services.github.GitHubService.get_session') + def test_update_webhook_value_error(self, session, mock_logger): + session().patch.side_effect = ValueError + self.service.update_webhook( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertIsNone(self.integration.secret) + mock_logger.exception.assert_called_with( + 'GitHub webhook update failed for project: %s', + self.project, + ) + + @mock.patch('readthedocs.oauth.services.github.log') + @mock.patch('readthedocs.oauth.services.github.GitHubService.get_session') + def test_get_provider_data_successful(self, session, mock_logger): + self.integration.provider_data = {} + self.integration.save() + + webhook_data = self.provider_data + rtd_webhook_url = 'https://{domain}{path}'.format( + domain=settings.PRODUCTION_DOMAIN, + path=reverse( + 'api_webhook', + kwargs={ + 'project_slug': self.project.slug, + 'integration_pk': self.integration.pk, + }, + ) + ) + webhook_data[0]["config"]["url"] = rtd_webhook_url + + session().get.return_value.status_code = 200 + session().get.return_value.json.return_value = webhook_data + + self.service.get_provider_data( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertEqual(self.integration.provider_data, webhook_data[0]) + mock_logger.info.assert_called_with( + 'GitHub integration updated with provider data for project: %s', + self.project, + ) + + @mock.patch('readthedocs.oauth.services.github.log') + @mock.patch('readthedocs.oauth.services.github.GitHubService.get_session') + def test_get_provider_data_404_error(self, session, mock_logger): + self.integration.provider_data = {} + self.integration.save() + + session().get.return_value.status_code = 404 + + self.service.get_provider_data( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertEqual(self.integration.provider_data, {}) + mock_logger.info.assert_called_with( + 'GitHub project does not exist or user does not have ' + 'permissions: project=%s', + self.project, + ) + + @mock.patch('readthedocs.oauth.services.github.log') + @mock.patch('readthedocs.oauth.services.github.GitHubService.get_session') + def test_get_provider_data_attribute_error(self, session, mock_logger): + self.integration.provider_data = {} + self.integration.save() + + session().get.side_effect = AttributeError + + self.service.get_provider_data( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertEqual(self.integration.provider_data, {}) + mock_logger.exception.assert_called_with( + 'GitHub webhook Listing failed for project: %s', + self.project, + ) + class BitbucketOAuthTests(TestCase): @@ -320,9 +540,32 @@ def setUp(self): self.client.login(username='eric', password='test') self.user = User.objects.get(pk=1) self.project = Project.objects.get(slug='pip') + self.project.repo = 'https://bitbucket.org/testuser/testrepo/' + self.project.save() self.org = RemoteOrganization.objects.create(slug='rtfd', json='') self.privacy = self.project.version_privacy_level self.service = BitbucketService(user=self.user, account=None) + self.integration = get( + GitHubWebhook, + project=self.project, + provider_data={ + 'links': { + 'self': { + 'href': 'https://bitbucket.org/' + } + } + } + ) + self.provider_data = { + 'values': [{ + 'links': { + 'self': { + 'href': 'https://bitbucket.org/' + } + }, + 'url': 'https://readthedocs.io/api/v2/webhook/test/99999999/', + },] + } def test_make_project_pass(self): repo = self.service.create_repository( @@ -390,6 +633,192 @@ def test_import_with_no_token(self): services = BitbucketService.for_user(self.user) self.assertEqual(services, []) + @mock.patch('readthedocs.oauth.services.bitbucket.log') + @mock.patch('readthedocs.oauth.services.bitbucket.BitbucketService.get_session') + def test_setup_webhook_successful(self, session, mock_logger): + session().post.return_value.status_code = 201 + session().post.return_value.json.return_value = {} + success, _ = self.service.setup_webhook( + self.project, + self.integration + ) + + self.assertTrue(success) + mock_logger.info.assert_called_with( + "Bitbucket webhook creation successful for project: %s", + self.project, + ) + + @mock.patch('readthedocs.oauth.services.bitbucket.log') + @mock.patch('readthedocs.oauth.services.bitbucket.BitbucketService.get_session') + def test_setup_webhook_404_error(self, session, mock_logger): + session().post.return_value.status_code = 404 + success, _ = self.service.setup_webhook( + self.project, + self.integration + ) + + self.assertFalse(success) + mock_logger.info.assert_called_with( + 'Bitbucket project does not exist or user does not have ' + 'permissions: project=%s', + self.project, + ) + + @mock.patch('readthedocs.oauth.services.bitbucket.log') + @mock.patch('readthedocs.oauth.services.bitbucket.BitbucketService.get_session') + def test_setup_webhook_value_error(self, session, mock_logger): + session().post.side_effect = ValueError + success = self.service.setup_webhook( + self.project, + self.integration + ) + + mock_logger.exception.assert_called_with( + 'Bitbucket webhook creation failed for project: %s', + self.project, + ) + + @mock.patch('readthedocs.oauth.services.bitbucket.log') + @mock.patch('readthedocs.oauth.services.bitbucket.BitbucketService.get_session') + def test_update_webhook_successful(self, session, mock_logger): + session().put.return_value.status_code = 200 + session().put.return_value.json.return_value = {} + success, _ = self.service.update_webhook( + self.project, + self.integration + ) + + self.assertTrue(success) + self.assertIsNotNone(self.integration.secret) + mock_logger.info.assert_called_with( + "Bitbucket webhook update successful for project: %s", + self.project, + ) + + @mock.patch('readthedocs.oauth.services.bitbucket.BitbucketService.get_session') + @mock.patch('readthedocs.oauth.services.bitbucket.BitbucketService.setup_webhook') + def test_update_webhook_404_error(self, setup_webhook, session): + session().put.return_value.status_code = 404 + self.service.update_webhook( + self.project, + self.integration + ) + + setup_webhook.assert_called_once_with( + self.project, + self.integration + ) + + @mock.patch('readthedocs.oauth.services.bitbucket.BitbucketService.get_session') + @mock.patch('readthedocs.oauth.services.bitbucket.BitbucketService.setup_webhook') + def test_update_webhook_no_provider_data(self, setup_webhook, session): + self.integration.provider_data = None + self.integration.save() + + session().put.side_effect = AttributeError + self.service.update_webhook( + self.project, + self.integration + ) + + setup_webhook.assert_called_once_with( + self.project, + self.integration + ) + + @mock.patch('readthedocs.oauth.services.bitbucket.log') + @mock.patch('readthedocs.oauth.services.bitbucket.BitbucketService.get_session') + def test_update_webhook_value_error(self, session, mock_logger): + session().put.side_effect = ValueError + self.service.update_webhook( + self.project, + self.integration + ) + + mock_logger.exception.assert_called_with( + 'Bitbucket webhook update failed for project: %s', + self.project, + ) + + @mock.patch('readthedocs.oauth.services.bitbucket.log') + @mock.patch('readthedocs.oauth.services.bitbucket.BitbucketService.get_session') + def test_get_provider_data_successful(self, session, mock_logger): + self.integration.provider_data = {} + self.integration.save() + + webhook_data = self.provider_data + rtd_webhook_url = 'https://{domain}{path}'.format( + domain=settings.PRODUCTION_DOMAIN, + path=reverse( + 'api_webhook', + kwargs={ + 'project_slug': self.project.slug, + 'integration_pk': self.integration.pk, + }, + ) + ) + webhook_data['values'][0]["url"] = rtd_webhook_url + + session().get.return_value.status_code = 200 + session().get.return_value.json.return_value = webhook_data + + self.service.get_provider_data( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertEqual(self.integration.provider_data, webhook_data['values'][0]) + mock_logger.info.assert_called_with( + 'Bitbucket integration updated with provider data for project: %s', + self.project, + ) + + @mock.patch('readthedocs.oauth.services.bitbucket.log') + @mock.patch('readthedocs.oauth.services.bitbucket.BitbucketService.get_session') + def test_get_provider_data_404_error(self, session, mock_logger): + self.integration.provider_data = {} + self.integration.save() + + session().get.return_value.status_code = 404 + + self.service.get_provider_data( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertEqual(self.integration.provider_data, {}) + mock_logger.info.assert_called_with( + 'Bitbucket project does not exist or user does not have ' + 'permissions: project=%s', + self.project, + ) + + @mock.patch('readthedocs.oauth.services.bitbucket.log') + @mock.patch('readthedocs.oauth.services.bitbucket.BitbucketService.get_session') + def test_get_provider_data_attribute_error(self, session, mock_logger): + self.integration.provider_data = {} + self.integration.save() + + session().get.side_effect = AttributeError + + self.service.get_provider_data( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertEqual(self.integration.provider_data, {}) + mock_logger.exception.assert_called_with( + 'Bitbucket webhook Listing failed for project: %s', + self.project, + ) + class GitLabOAuthTests(TestCase): @@ -488,6 +917,8 @@ def setUp(self): self.client.login(username='eric', password='test') self.user = User.objects.get(pk=1) self.project = Project.objects.get(slug='pip') + self.project.repo = 'https://gitlab.com/testorga/testrepo' + self.project.save() self.org = RemoteOrganization.objects.create(slug='testorga', json='') self.privacy = self.project.version_privacy_level self.service = GitLabService(user=self.user, account=None) @@ -495,6 +926,19 @@ def setUp(self): self.external_build = get( Build, project=self.project, version=self.external_version ) + self.integration = get( + GitLabWebhook, + project=self.project, + provider_data={ + 'id': '999999999' + } + ) + self.provider_data = [ + { + 'id': 1084320, + 'url': 'https://readthedocs.io/api/v2/webhook/test/99999999/', + } + ] def get_private_repo_data(self): """Manipulate repo response data to get private repo data.""" @@ -568,11 +1012,6 @@ def test_make_private_project(self): repo = self.service.create_repository(data, organization=self.org) self.assertIsNotNone(repo) - def test_setup_webhook(self): - success, response = self.service.setup_webhook(self.project) - self.assertFalse(success) - self.assertIsNone(response) - @mock.patch('readthedocs.oauth.services.gitlab.log') @mock.patch('readthedocs.oauth.services.gitlab.GitLabService.get_session') @mock.patch('readthedocs.oauth.services.gitlab.GitLabService._get_repo_id') @@ -629,3 +1068,211 @@ def test_send_build_status_value_error(self, repo_id, session, mock_logger): 'GitLab commit status creation failed for project: %s', self.project, ) + + @mock.patch('readthedocs.oauth.services.gitlab.log') + @mock.patch('readthedocs.oauth.services.gitlab.GitLabService.get_session') + def test_setup_webhook_successful(self, session, mock_logger): + session().post.return_value.status_code = 201 + session().post.return_value.json.return_value = {} + success, _ = self.service.setup_webhook( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertTrue(success) + self.assertIsNotNone(self.integration.secret) + mock_logger.info.assert_called_with( + "GitLab webhook creation successful for project: %s", + self.project, + ) + + @mock.patch('readthedocs.oauth.services.gitlab.log') + @mock.patch('readthedocs.oauth.services.gitlab.GitLabService.get_session') + def test_setup_webhook_404_error(self, session, mock_logger): + session().post.return_value.status_code = 404 + success, _ = self.service.setup_webhook( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertFalse(success) + self.assertIsNone(self.integration.secret) + mock_logger.info.assert_called_with( + 'Gitlab project does not exist or user does not have ' + 'permissions: project=%s', + self.project, + ) + + @mock.patch('readthedocs.oauth.services.gitlab.log') + @mock.patch('readthedocs.oauth.services.gitlab.GitLabService.get_session') + def test_setup_webhook_value_error(self, session, mock_logger): + session().post.side_effect = ValueError + success = self.service.setup_webhook( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertIsNone(self.integration.secret) + mock_logger.exception.assert_called_with( + 'GitLab webhook creation failed for project: %s', + self.project, + ) + + @mock.patch('readthedocs.oauth.services.gitlab.log') + @mock.patch('readthedocs.oauth.services.gitlab.GitLabService.get_session') + @mock.patch('readthedocs.oauth.services.gitlab.GitLabService._get_repo_id') + def test_update_webhook_successful(self, repo_id, session, mock_logger): + repo_id.return_value = '9999' + session().put.return_value.status_code = 200 + session().put.return_value.json.return_value = {} + success, _ = self.service.update_webhook( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertTrue(success) + self.assertIsNotNone(self.integration.secret) + mock_logger.info.assert_called_with( + "GitLab webhook update successful for project: %s", + self.project, + ) + + @mock.patch('readthedocs.oauth.services.gitlab.GitLabService.get_session') + @mock.patch('readthedocs.oauth.services.gitlab.GitLabService.setup_webhook') + @mock.patch('readthedocs.oauth.services.gitlab.GitLabService._get_repo_id') + def test_update_webhook_404_error(self, repo_id, setup_webhook, session): + repo_id.return_value = '9999' + session().put.return_value.status_code = 404 + self.service.update_webhook( + self.project, + self.integration + ) + + setup_webhook.assert_called_once_with( + self.project, + self.integration + ) + + @mock.patch('readthedocs.oauth.services.gitlab.GitLabService.get_session') + @mock.patch('readthedocs.oauth.services.gitlab.GitLabService.setup_webhook') + @mock.patch('readthedocs.oauth.services.gitlab.GitLabService._get_repo_id') + def test_update_webhook_no_provider_data(self, repo_id, setup_webhook, session): + self.integration.provider_data = None + self.integration.save() + + repo_id.return_value = '9999' + session().put.side_effect = AttributeError + self.service.update_webhook( + self.project, + self.integration + ) + + setup_webhook.assert_called_once_with( + self.project, + self.integration + ) + + @mock.patch('readthedocs.oauth.services.gitlab.log') + @mock.patch('readthedocs.oauth.services.gitlab.GitLabService.get_session') + @mock.patch('readthedocs.oauth.services.gitlab.GitLabService._get_repo_id') + def test_update_webhook_value_error(self, repo_id, session, mock_logger): + repo_id.return_value = '9999' + session().put.side_effect = ValueError + self.service.update_webhook( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertIsNone(self.integration.secret) + mock_logger.exception.assert_called_with( + 'GitLab webhook update failed for project: %s', + self.project, + ) + + @mock.patch('readthedocs.oauth.services.gitlab.log') + @mock.patch('readthedocs.oauth.services.gitlab.GitLabService.get_session') + def test_get_provider_data_successful(self, session, mock_logger): + self.integration.provider_data = {} + self.integration.save() + + webhook_data = self.provider_data + rtd_webhook_url = 'https://{domain}{path}'.format( + domain=settings.PRODUCTION_DOMAIN, + path=reverse( + 'api_webhook', + kwargs={ + 'project_slug': self.project.slug, + 'integration_pk': self.integration.pk, + }, + ) + ) + webhook_data[0]["url"] = rtd_webhook_url + + session().get.return_value.status_code = 200 + session().get.return_value.json.return_value = webhook_data + + self.service.get_provider_data( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertEqual(self.integration.provider_data, webhook_data[0]) + mock_logger.info.assert_called_with( + 'GitLab integration updated with provider data for project: %s', + self.project, + ) + + @mock.patch('readthedocs.oauth.services.gitlab.log') + @mock.patch('readthedocs.oauth.services.gitlab.GitLabService.get_session') + def test_get_provider_data_404_error(self, session, mock_logger): + self.integration.provider_data = {} + self.integration.save() + + session().get.return_value.status_code = 404 + + self.service.get_provider_data( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertEqual(self.integration.provider_data, {}) + mock_logger.info.assert_called_with( + 'GitLab project does not exist or user does not have ' + 'permissions: project=%s', + self.project, + ) + + @mock.patch('readthedocs.oauth.services.gitlab.log') + @mock.patch('readthedocs.oauth.services.gitlab.GitLabService.get_session') + def test_get_provider_data_attribute_error(self, session, mock_logger): + self.integration.provider_data = {} + self.integration.save() + + session().get.side_effect = AttributeError + + self.service.get_provider_data( + self.project, + self.integration + ) + + self.integration.refresh_from_db() + + self.assertEqual(self.integration.provider_data, {}) + mock_logger.exception.assert_called_with( + 'GitLab webhook Listing failed for project: %s', + self.project, + ) diff --git a/readthedocs/templates/projects/integration_webhook_detail.html b/readthedocs/templates/projects/integration_webhook_detail.html index d8911172884..4fdebcd4090 100644 --- a/readthedocs/templates/projects/integration_webhook_detail.html +++ b/readthedocs/templates/projects/integration_webhook_detail.html @@ -19,9 +19,8 @@ {% if integration.has_sync and integration.can_sync %}

{% blocktrans trimmed %} - This webhook was configured when this project was imported - or it was automatically created with the correct configuration. If this - integration is not functioning correctly, try re-syncing the webhook: + This integration is being managed automatically by Read the Docs. If + it isn't functioning correctly, try re-syncing the webhook: {% endblocktrans %}

@@ -40,8 +39,8 @@ {% if integration.has_sync and not integration.can_sync %}

{% blocktrans trimmed %} - This webhook was created automatically from an existing webhook - configured on your repository. If this integration is not functioning correctly, + This integration is not managed by Read the Docs currently. + If this integration is not functioning correctly, you can try re-syncing it. Otherwise you'll need to update the configuration on your repository. You can use the following address to manually configure this webhook.