Enable token auth sync

CHANGES/6540.feature

@@ -0,0 +1,2 @@
+Enable token authentication for syncing Collections.
+Added `auth_url` and `token` `fields <https://docs.ansible.com/ansible/latest/user_guide/collections_using.html#configuring-the-ansible-galaxy-client>`_ to `CollectionRemote`

docs/_scripts/remote-collection-token.sh

@@ -0,0 +1,13 @@
+# Create a remote that syncs some versions of django into your repository.
+http POST $BASE_ADDR/pulp/api/v3/remotes/ansible/collection/ \
+    name='bar' \
+    auth_url='https://sso.qa.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token' \
+    token=$ANSIBLE_TOKEN_AUTH \
+    tls_validation=false \
+    url='https://cloud.redhat.com/api/automation-hub/v3/collections/testing/ansible_testing_content'
+
+# Export an environment variable for the new remote URI.
+export REMOTE_HREF=$(http $BASE_ADDR/pulp/api/v3/remotes/ansible/collection/ | jq -r '.results[] | select(.name == "bar") | .pulp_href')
+
+# Lets inspect our newly created Remote
+http $BASE_ADDR$REMOTE_HREF

docs/workflows/collections.rst

@@ -136,6 +136,31 @@ Remote GET Response::
         "url": "https://galaxy-dev.ansible.com/api/v2/collections/testing/ansible_testing_content/",
     }
 
+For `remote sources that require authentication <https://docs.ansible.com/ansible/latest/user_guide/collections_using.html#configuring-the-ansible-galaxy-client>`_, tokens can be used. You can provide the ``token``
+and/or ``auth_url``.
+
+In this example we will be syncing the Collection with ``namespace=testing``  and ``name=ansible_testing_content``
+from ``https://cloud.redhat.com/api/automation-hub/v3/collections/testing/ansible_testing_content``.
+
+.. literalinclude:: ../_scripts/remote-collection-token.sh
+   :language: bash
+
+Remote GET Response::
+
+    {
+        "pulp_created": "2019-04-29T13:51:10.860792Z",
+        "pulp_href": "/pulp/api/v3/remotes/ansible/collection/e1c65074-3a4f-4f06-837e-75a9a90f2c31/",
+        "pulp_last_updated": "2019-04-29T13:51:10.860805Z",
+        "download_concurrency": 20,
+        "name": "bar",
+        "policy": "immediate",
+        "proxy_url": null,
+        "ssl_ca_certificate": null,
+        "ssl_client_certificate": null,
+        "ssl_client_key": null,
+        "ssl_validation": true,
+        "url": "https://galaxy-dev.ansible.com/api/v2/collections/testing/ansible_testing_content/",
+    }
 
 Sync Repository foo with CollectionRemote
 -----------------------------------------

pulp_ansible/app/downloaders.py

@@ -0,0 +1,128 @@
+from logging import getLogger
+import asyncio
+import backoff
+import json
+
+from aiohttp import BasicAuth
+from aiohttp.client_exceptions import ClientResponseError
+
+from pulpcore.plugin.download import http_giveup, DownloaderFactory, HttpDownloader
+
+
+log = getLogger(__name__)
+
+
+class TokenAuthHttpDownloader(HttpDownloader):
+    """
+    Custom Downloader that automatically handles Token Based and Basic Authentication.
+    """
+
+    TOKEN_LOCK = asyncio.Lock()
+
+    def __init__(self, url, auth_url, token, **kwargs):
+        """
+        Initialize the downloader.
+        """
+        self.ansible_auth_url = auth_url
+        self.ansible_token = token
+        super().__init__(url, **kwargs)
+
+    @backoff.on_exception(backoff.expo, ClientResponseError, max_tries=10, giveup=http_giveup)
+    async def _run(self, extra_data=None):
+        """
+        Download, validate, and compute digests on the `url`. This is a coroutine.
+
+        This method is decorated with a backoff-and-retry behavior to retry HTTP 429 errors. It
+        retries with exponential backoff 10 times before allowing a final exception to be raised.
+
+        This method provides the same return object type and documented in
+        :meth:`~pulpcore.plugin.download.BaseDownloader._run`.
+
+        Ansible token reference:
+            https://github.com/ansible/ansible/blob/devel/lib/ansible/galaxy/token.py
+
+        """
+        if not self.ansible_token:
+            # No Token
+            return await super()._run(extra_data=extra_data)
+
+        if not self.ansible_auth_url:
+            # Galaxy Token
+            headers = {"Authorization": self.ansible_token}
+        else:
+            token = await self.update_token_from_auth_url()
+            # Keycloak Token
+            headers = {"Authorization": "Bearer {token}".format(token=token)}
+
+        async with self.session.get(self.url, headers=headers, proxy=self.proxy) as response:
+            response.raise_for_status()
+            to_return = await self._handle_response(response)
+            await response.release()
+
+        if self._close_session_on_finalize:
+            self.session.close()
+        return to_return
+
+    async def update_token_from_auth_url(self):
+        """
+        Update the Bearer token to be used with all requests.
+        """
+        async with self.TOKEN_LOCK:
+            log.info("Updating bearer token")
+            form_payload = {
+                "grant_type": "refresh_token",
+                "client_id": "cloud-services",
+                "refresh_token": self.ansible_token,
+            }
+            url = self.ansible_auth_url
+            async with self.session.post(url, data=form_payload, raise_for_status=True) as response:
+                token_data = await response.text()
+
+            return json.loads(token_data)["access_token"]
+
+
+class AnsibleDownloaderFactory(DownloaderFactory):
+    """A factory for creating downloader objects that are configured from with remote settings."""
+
+    def __init__(self, remote, downloader_overrides=None):
+        """
+        Initialize AnsibleDownloaderFactory.
+
+        Args:
+            remote (:class:`~pulpcore.plugin.models.Remote`): The remote used to populate
+                downloader settings.
+            downloader_overrides (dict): Keyed on a scheme name, e.g. 'https' or 'ftp' and the value
+                is the downloader class to be used for that scheme, e.g.
+                {'https': MyCustomDownloader}. These override the default values.
+        """
+        if not downloader_overrides:
+            downloader_overrides = {
+                "http": TokenAuthHttpDownloader,
+                "https": TokenAuthHttpDownloader,
+            }
+        super().__init__(remote, downloader_overrides)
+
+    def _http_or_https(self, download_class, url, **kwargs):
+        """
+        Build a downloader for http:// or https:// URLs.
+
+        Args:
+            download_class (:class:`~pulpcore.plugin.download.BaseDownloader`): The download
+                class to be instantiated.
+            url (str): The download URL.
+            kwargs (dict): All kwargs are passed along to the downloader. At a minimum, these
+                include the :class:`~pulpcore.plugin.download.BaseDownloader` parameters.
+
+        Returns:
+            :class:`~pulpcore.plugin.download.HttpDownloader`: A downloader that
+            is configured with the remote settings.
+
+        """
+        options = {"session": self._session}
+        if self._remote.proxy_url:
+            options["proxy"] = self._remote.proxy_url
+
+        if not self._remote.token and self._remote.username and self._remote.password:
+            options["auth"] = BasicAuth(login=self._remote.username, password=self._remote.password)
+
+        return download_class(url, self._remote.auth_url, self._remote.token, **options, **kwargs)

pulp_ansible/app/migrations/0019_collection_token.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.13 on 2020-06-09 21:13
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ansible', '0018_fix_collection_relative_path'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='collectionremote',
+            name='auth_url',
+            field=models.CharField(max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='collectionremote',
+            name='token',
+            field=models.TextField(max_length=2000, null=True),
+        ),
+    ]

pulp_ansible/app/models.py

@@ -13,6 +13,7 @@
     RepositoryVersionDistribution,
     Task,
 )
+from .downloaders import AnsibleDownloaderFactory
 
 log = getLogger(__name__)
 
@@ -227,6 +228,29 @@ class CollectionRemote(Remote):
     TYPE = "collection"
 
     requirements_file = models.TextField(null=True, max_length=255)
+    auth_url = models.CharField(null=True, max_length=255)
+    token = models.TextField(null=True, max_length=2000)
+
+    @property
+    def download_factory(self):
+        """
+        Return the DownloaderFactory which can be used to generate asyncio capable downloaders.
+
+        Upon first access, the DownloaderFactory is instantiated and saved internally.
+
+        Plugin writers are expected to override when additional configuration of the
+        DownloaderFactory is needed.
+
+        Returns:
+            DownloadFactory: The instantiated DownloaderFactory to be used by
+                get_downloader()
+
+        """
+        try:
+            return self._download_factory
+        except AttributeError:
+            self._download_factory = AnsibleDownloaderFactory(self)
+            return self._download_factory
 
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"

pulp_ansible/app/serializers.py

@@ -95,6 +95,22 @@ class CollectionRemoteSerializer(RemoteSerializer):
         required=False,
         allow_null=True,
     )
+    auth_url = serializers.CharField(
+        help_text=_("The URL to receive a session token from, e.g. used with Automation Hub."),
+        allow_null=True,
+        required=False,
+        max_length=255,
+    )
+    token = serializers.CharField(
+        help_text=_(
+            "The token key to use for authentication. See https://docs.ansible.com/ansible/"
+            "latest/user_guide/collections_using.html#configuring-the-ansible-galaxy-client"
+            "for more details"
+        ),
+        allow_null=True,
+        required=False,
+        max_length=2000,
+    )
 
     def validate(self, data):
         """
@@ -124,7 +140,7 @@ def validate(self, data):
         return data
 
     class Meta:
-        fields = RemoteSerializer.Meta.fields + ("requirements_file",)
+        fields = RemoteSerializer.Meta.fields + ("requirements_file", "auth_url", "token")
         model = CollectionRemote
 
 

pulp_ansible/tests/functional/cli/test_collection_install.py

@@ -1,5 +1,5 @@
 """Tests that Collections hosted by Pulp can be installed by ansible-galaxy."""
-
+import os
 from os import path
 import subprocess
 import tempfile
@@ -14,14 +14,18 @@
 from pulp_smash.pulp3.utils import gen_distribution, gen_repo
 
 from pulp_ansible.tests.functional.constants import (
+    AH_AUTH_URL,
     ANSIBLE_COLLECTION_TESTING_URL_V2,
     ANSIBLE_DEMO_COLLECTION,
+    TOKEN_DEMO_COLLECTION,
+    TOKEN_AUTH_COLLECTION_TESTING_URL,
 )
 from pulp_ansible.tests.functional.utils import (
     gen_ansible_client,
     gen_ansible_remote,
     monitor_task,
 )
+from pulp_ansible.tests.functional.utils import skip_if
 from pulp_ansible.tests.functional.utils import set_up_module as setUpModule  # noqa:F401
 
 
@@ -31,17 +35,18 @@ class InstallCollectionTestCase(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         """Create class-wide variables."""
+        cls.AH_token = "AUTOMATION_HUB_TOKEN_AUTH" in os.environ
+        cls.GH_token = "GITHUB_API_KEY" in os.environ
         cls.client = gen_ansible_client()
         cls.repo_api = RepositoriesAnsibleApi(cls.client)
         cls.remote_collection_api = RemotesCollectionApi(cls.client)
         cls.distributions_api = DistributionsAnsibleApi(cls.client)
 
-    def test_install_collection(self):
-        """Test whether ansible-galaxy can install a Collection hosted by Pulp."""
+    def create_install_scenario(self, body, collection_name):
+        """Create Install scenario."""
         repo = self.repo_api.create(gen_repo())
         self.addCleanup(self.repo_api.delete, repo.pulp_href)
 
-        body = gen_ansible_remote(url=ANSIBLE_COLLECTION_TESTING_URL_V2)
         remote = self.remote_collection_api.create(body)
         self.addCleanup(self.remote_collection_api.delete, remote.pulp_href)
 
@@ -63,11 +68,11 @@ def test_install_collection(self):
 
         with tempfile.TemporaryDirectory() as temp_dir:
             cmd = "ansible-galaxy collection install {} -c -s {} -p {}".format(
-                ANSIBLE_DEMO_COLLECTION, distribution.client_url, temp_dir
+                collection_name, distribution.client_url, temp_dir
             )
 
             directory = "{}/ansible_collections/{}".format(
-                temp_dir, ANSIBLE_DEMO_COLLECTION.replace(".", "/")
+                temp_dir, collection_name.replace(".", "/")
             )
 
             self.assertTrue(
@@ -77,3 +82,29 @@ def test_install_collection(self):
             subprocess.run(cmd.split())
 
             self.assertTrue(path.exists(directory), "Could not find directory {}".format(directory))
+
+    def test_install_collection(self):
+        """Test whether ansible-galaxy can install a Collection hosted by Pulp."""
+        body = gen_ansible_remote(url=ANSIBLE_COLLECTION_TESTING_URL_V2)
+        self.create_install_scenario(body, ANSIBLE_DEMO_COLLECTION)
+
+    @skip_if(bool, "AH_token", False)
+    def test_install_collection_with_token_from_automation_hub(self):
+        """Test whether ansible-galaxy can install a Collection hosted by Pulp."""
+        body = gen_ansible_remote(
+            url=TOKEN_AUTH_COLLECTION_TESTING_URL,
+            auth_url=AH_AUTH_URL,
+            token=os.environ["AUTOMATION_HUB_TOKEN_AUTH"],
+            tls_validation=False,
+        )
+        self.create_install_scenario(body, TOKEN_DEMO_COLLECTION)
+
+    @skip_if(bool, "GH_token", False)
+    def test_install_collection_with_token_from_galaxy(self):
+        """Test whether ansible-galaxy can install a Collection hosted by Pulp."""
+        token = os.environ["GITHUB_API_KEY"]
+        PULP_COLLECTION = ANSIBLE_COLLECTION_TESTING_URL_V2.replace("testing", "pulp")
+        PULP_COLLECTION = PULP_COLLECTION.replace("k8s_demo_collection", "pulp_installer")
+
+        body = gen_ansible_remote(url=PULP_COLLECTION, token=token)
+        self.create_install_scenario(body, "pulp.pulp_installer")

pulp_ansible/tests/functional/constants.py

@@ -8,6 +8,8 @@
     BASE_CONTENT_PATH,
 )
 
+AH_AUTH_URL = "https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token"
+
 GALAXY_ANSIBLE_BASE_URL = "https://galaxy.ansible.com"
 
 ANSIBLE_ROLE_NAME = "ansible.role"
@@ -55,6 +57,8 @@
 
 ANSIBLE_DEMO_COLLECTION = "testing.k8s_demo_collection"
 
+TOKEN_DEMO_COLLECTION = "ansible.posix"
+
 ANSIBLE_COLLECTION_CONTENT_NAME = "ansible.collection_version"
 
 ANSIBLE_COLLECTION_FIXTURE_COUNT = 1
@@ -82,6 +86,10 @@
 - testing.k8s_demo_collection
 """
 
+TOKEN_AUTH_COLLECTIONS_URL = "https://cloud.redhat.com/api/automation-hub/v3/collections/"
+
+TOKEN_AUTH_COLLECTION_TESTING_URL = urljoin(TOKEN_AUTH_COLLECTIONS_URL, "ansible/posix")
+
 # Ansible Galaxy V2 Endpoints
 
 ANSIBLE_GALAXY_COLLECTION_URL_V2 = urljoin(GALAXY_ANSIBLE_BASE_URL, "api/v2/collections/")