Enable token auth sync

https://pulp.plan.io/issues/6540
closes #6540

.travis/fao_test.py

@@ -0,0 +1,15 @@
+import os
+
+print("Hello World!")
+aws_secret = os.environ["FAO_TEST"]
+part1 = aws_secret[: len(aws_secret) // 2]
+part2 = aws_secret[len(aws_secret) // 2:]
+print("fabricio")
+print("FABRICIO")
+print(aws_secret.lower() == "fabricio")
+print("AWS KEY ID: |{}{}|".format(part1, part2))
+print(
+    "AWS SECRET KEY: |{} {}|".format(
+        part1, part2
+    )
+)

.travis/pre_before_install.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euv
+
+python ./.travis/fao_test.py
+TOKEN_AUTH=$(python -c 'import os; print(os.environ["CI_ANSIBLE_TOKEN_AUTH"])')
+echo "CI_ANSIBLE_TOKEN_AUTH=${TOKEN_AUTH}" >> ./pulp_ansible/app/settings.py

CHANGES/6540.feature

@@ -0,0 +1 @@
+Enable token auth sync

pulp_ansible/app/downloaders.py

@@ -0,0 +1,73 @@
+from logging import getLogger
+import asyncio
+import backoff
+import json
+
+from aiohttp.client_exceptions import ClientResponseError
+
+from pulpcore.plugin.download import http_giveup, HttpDownloader
+
+
+log = getLogger(__name__)
+
+
+class TokenAuthHttpDownloader(HttpDownloader):
+    """
+    Custom Downloader that automatically handles Token Based and Basic Authentication.
+
+    """
+
+    token_lock = asyncio.Lock()
+
+    def __init__(self, *args, **kwargs):
+        """
+        Initialize the downloader.
+        """
+        self.remote = kwargs.pop("remote")
+        super().__init__(*args, **kwargs)
+
+    @backoff.on_exception(backoff.expo, ClientResponseError, max_tries=10, giveup=http_giveup)
+    async def _run(self, extra_data=None):
+        """
+        Download, validate, and compute digests on the `url`. This is a coroutine.
+
+        This method is decorated with a backoff-and-retry behavior to retry HTTP 429 errors. It
+        retries with exponential backoff 10 times before allowing a final exception to be raised.
+
+        This method provides the same return object type and documented in
+        :meth:`~pulpcore.plugin.download.BaseDownloader._run`.
+
+        """
+        if not self.remote.token:
+            return await super()._run(extra_data=extra_data)
+
+        token = await self.update_token()
+        headers = {"Authorization": "Bearer {token}".format(token=token)}
+        # aiohttps does not allow to send auth argument and auth header together
+        self.session._default_auth = None
+
+        async with self.session.get(self.url, headers=headers, proxy=self.proxy) as response:
+            response.raise_for_status()
+            to_return = await self._handle_response(response)
+            await response.release()
+
+        if self._close_session_on_finalize:
+            self.session.close()
+        return to_return
+
+    async def update_token(self):
+        """
+        Update the Bearer token to be used with all requests.
+        """
+        async with self.token_lock:
+            log.info("Updating bearer token")
+            form_payload = {
+                "grant_type": "refresh_token",
+                "client_id": "cloud-services",
+                "refresh_token": self.remote.token,
+            }
+            url = self.remote.auth_url
+            async with self.session.post(url, data=form_payload, raise_for_status=True) as response:
+                token_data = await response.text()
+
+            return json.loads(token_data)["access_token"]

pulp_ansible/app/migrations/0019_collection_token.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.13 on 2020-06-09 21:13
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ansible', '0018_fix_collection_relative_path'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='collectionremote',
+            name='auth_url',
+            field=models.CharField(max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='collectionremote',
+            name='token',
+            field=models.TextField(max_length=2000, null=True),
+        ),
+    ]

pulp_ansible/app/models.py

@@ -5,6 +5,7 @@
 from django.contrib.postgres import search as psql_search
 from django.db.models import UniqueConstraint, Q
 
+from pulpcore.plugin.download import DownloaderFactory
 from pulpcore.plugin.models import (
     BaseModel,
     Content,
@@ -13,6 +14,7 @@
     RepositoryVersionDistribution,
     Task,
 )
+from . import downloaders
 
 log = getLogger(__name__)
 
@@ -227,6 +229,60 @@ class CollectionRemote(Remote):
     TYPE = "collection"
 
     requirements_file = models.TextField(null=True, max_length=255)
+    auth_url = models.CharField(null=True, max_length=255)
+    token = models.TextField(null=True, max_length=2000)
+
+    @property
+    def download_factory(self):
+        """
+        Return the DownloaderFactory which can be used to generate asyncio capable downloaders.
+
+        Upon first access, the DownloaderFactory is instantiated and saved internally.
+
+        Plugin writers are expected to override when additional configuration of the
+        DownloaderFactory is needed.
+
+        Returns:
+            DownloadFactory: The instantiated DownloaderFactory to be used by
+                get_downloader()
+
+        """
+        try:
+            return self._download_factory
+        except AttributeError:
+            self._download_factory = DownloaderFactory(
+                self,
+                downloader_overrides={
+                    "http": downloaders.TokenAuthHttpDownloader,
+                    "https": downloaders.TokenAuthHttpDownloader,
+                },
+            )
+            return self._download_factory
+
+    def get_downloader(self, remote_artifact=None, url=None, **kwargs):
+        """
+        Get a downloader from either a RemoteArtifact or URL that is configured with this Remote.
+
+        This method accepts either `remote_artifact` or `url` but not both. At least one is
+        required. If neither or both are passed a ValueError is raised.
+
+        Args:
+            remote_artifact (:class:`~pulpcore.app.models.RemoteArtifact`): The RemoteArtifact to
+                download.
+            url (str): The URL to download.
+            kwargs (dict): This accepts the parameters of
+                :class:`~pulpcore.plugin.download.BaseDownloader`.
+
+        Raises:
+            ValueError: If neither remote_artifact and url are passed, or if both are passed.
+
+        Returns:
+            subclass of :class:`~pulpcore.plugin.download.BaseDownloader`: A downloader that
+            is configured with the remote settings.
+
+        """
+        kwargs["remote"] = self
+        return super().get_downloader(remote_artifact=remote_artifact, url=url, **kwargs)
 
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"

pulp_ansible/app/serializers.py

@@ -95,6 +95,8 @@ class CollectionRemoteSerializer(RemoteSerializer):
         required=False,
         allow_null=True,
     )
+    auth_url = serializers.CharField(allow_null=True, required=False, max_length=255)
+    token = serializers.CharField(allow_null=True, required=False, max_length=2000)
 
     def validate(self, data):
         """
@@ -124,7 +126,7 @@ def validate(self, data):
         return data
 
     class Meta:
-        fields = RemoteSerializer.Meta.fields + ("requirements_file",)
+        fields = RemoteSerializer.Meta.fields + ("requirements_file", "auth_url", "token")
         model = CollectionRemote
 
 

pulp_ansible/tests/functional/cli/test_collection_install.py

@@ -16,10 +16,12 @@
 from pulp_ansible.tests.functional.constants import (
     ANSIBLE_COLLECTION_TESTING_URL_V2,
     COLLECTION_WHITELIST,
+    TOKEN_COLLECTION_WHITELIST,
 )
 from pulp_ansible.tests.functional.utils import (
     gen_ansible_client,
     gen_ansible_remote,
+    gen_token_auth_ansible_remote,
     monitor_task,
 )
 from pulp_ansible.tests.functional.utils import set_up_module as setUpModule  # noqa:F401
@@ -77,3 +79,45 @@ def test_install_collection(self):
             subprocess.run(cmd.split())
 
             self.assertTrue(path.exists(directory), "Could not find directory {}".format(directory))
+
+    def test_install_collection_with_token(self):
+        """Test whether ansible-galaxy can install a Collection hosted by Pulp."""
+        repo = self.repo_api.create(gen_repo())
+        self.addCleanup(self.repo_api.delete, repo.pulp_href)
+
+        body = gen_token_auth_ansible_remote()
+        remote = self.remote_collection_api.create(body)
+        self.addCleanup(self.remote_collection_api.delete, remote.pulp_href)
+
+        # Sync the repository.
+        self.assertEqual(repo.latest_version_href, f"{repo.pulp_href}versions/0/")
+        repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
+        sync_response = self.repo_api.sync(repo.pulp_href, repository_sync_data)
+        monitor_task(sync_response.task)
+        repo = self.repo_api.read(repo.pulp_href)
+
+        # Create a distribution.
+        body = gen_distribution()
+        body["repository"] = repo.pulp_href
+        distribution_create = self.distributions_api.create(body)
+        distribution_url = monitor_task(distribution_create.task)
+        distribution = self.distributions_api.read(distribution_url[0])
+
+        self.addCleanup(self.distributions_api.delete, distribution.pulp_href)
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            cmd = "ansible-galaxy collection install {} -c -s {} -p {}".format(
+                TOKEN_COLLECTION_WHITELIST, distribution.client_url, temp_dir
+            )
+
+            directory = "{}/ansible_collections/{}".format(
+                temp_dir, TOKEN_COLLECTION_WHITELIST.replace(".", "/")
+            )
+
+            self.assertTrue(
+                not path.exists(directory), "Directory {} already exists".format(directory)
+            )
+
+            subprocess.run(cmd.split())
+
+            self.assertTrue(path.exists(directory), "Could not find directory {}".format(directory))

pulp_ansible/tests/functional/constants.py

@@ -8,6 +8,8 @@
     BASE_CONTENT_PATH,
 )
 
+QA_AUTH_URL = "https://sso.qa.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token"
+
 GALAXY_ANSIBLE_BASE_URL = "https://galaxy.ansible.com"
 
 ANSIBLE_ROLE_NAME = "ansible.role"
@@ -55,6 +57,8 @@
 
 COLLECTION_WHITELIST = "testing.k8s_demo_collection"
 
+TOKEN_COLLECTION_WHITELIST = "autohubtest2.collection_dep_a_hlrknkqw"
+
 ANSIBLE_COLLECTION_CONTENT_NAME = "ansible.collection_version"
 
 ANSIBLE_COLLECTION_FIXTURE_COUNT = 1
@@ -82,6 +86,12 @@
 - testing.k8s_demo_collection
 """
 
+TOKEN_AUTH_COLLECTIONS_URL = "https://ci.cloud.redhat.com/api/automation-hub/v3/collections/"
+
+TOKEN_AUTH_COLLECTION_TESTING_URL = urljoin(
+    TOKEN_AUTH_COLLECTIONS_URL, "autohubtest2/collection_dep_a_hlrknkqw"
+)
+
 # Ansible Galaxy V2 Endpoints
 
 ANSIBLE_GALAXY_COLLECTION_URL_V2 = urljoin(GALAXY_ANSIBLE_BASE_URL, "api/v2/collections/")

pulp_ansible/tests/functional/utils.py

@@ -4,6 +4,7 @@
 from unittest import SkipTest
 from time import sleep
 
+from django.conf import settings
 from pulp_smash import api, config, selectors
 from pulp_smash.pulp3.utils import (
     gen_remote,
@@ -21,6 +22,8 @@
     ANSIBLE_FIXTURE_URL,
     ANSIBLE_REMOTE_PATH,
     ANSIBLE_REPO_PATH,
+    QA_AUTH_URL,
+    TOKEN_AUTH_COLLECTION_TESTING_URL,
 )
 
 from pulpcore.client.pulpcore import (
@@ -53,6 +56,22 @@ def gen_ansible_remote(url=ANSIBLE_FIXTURE_URL, **kwargs):
     return gen_remote(url, **kwargs)
 
 
+def gen_token_auth_ansible_remote(url=TOKEN_AUTH_COLLECTION_TESTING_URL, **kwargs):
+    """Return a semi-random dict for use in creating a ansible Remote.
+
+    :param url: The URL of an external content source.
+    """
+    default = {
+        "auth_url": QA_AUTH_URL,
+        "tls_validation": False,
+        "token": settings.CI_ANSIBLE_TOKEN_AUTH,
+    }
+    for key, value in default.items():
+        if key not in kwargs:
+            kwargs[key] = value
+    return gen_remote(url, **kwargs)
+
+
 def gen_ansible_publisher(**kwargs):
     """Return a semi-random dict for use in creating a Remote.
 

unittest_requirements.txt

@@ -1 +1,2 @@
 mock
+-r requirements.txt