Add ability to synchronize Oracle ULN repositories

fixes #7905
https://pulp.plan.io/issues/7905

Should also work for non-Oracle ULN repositories, tested using Oracle.

CHANGES/7905.feature

@@ -0,0 +1,2 @@
+Added the ability to synchronize Oracle ULN repositories using ULN remotes.
+You can set an instance wide ULN server base URL using the DEFAULT_ULN_SERVER_BASE_URL setting.

coverage.md

@@ -17,6 +17,7 @@ This file contains list of features and their test coverage.
    | only types contained in rpm-unsigned |
 | As a user, I can sync and skip specific type (srpm) | NO |  |
 | As a user, I can sync opensuse repository | NO |  |
+| As a user, I can sync Oracle repository using ULN | NO | |
 | As a user, I can sync from a mirror list | YES |  |
 | As a user, I can sync from a mirror list with comments | YES |  |
 | As a user, I can sync from CDN using certificates | YES |  |

pulp_rpm/app/downloaders.py

@@ -1,6 +1,12 @@
+import os
+
+from aiohttp_xmlrpc.client import ServerProxy
 from logging import getLogger
+from lxml import etree
+from urllib.parse import urlparse
 
 from pulpcore.plugin.download import FileDownloader, HttpDownloader
+from pulp_rpm.app.exceptions import UlnCredentialsError
 from pulp_rpm.app.shared_utils import urlpath_sanitize
 
 
@@ -37,7 +43,6 @@ class RpmDownloader(HttpDownloader):
 
     Raises:
         FileNotFoundError: If aiohttp response status is 404 and silenced.
-
     """
 
     def __init__(
@@ -61,7 +66,6 @@ def raise_for_status(self, response):
         Raises:
             FileNotFoundError: If aiohttp response status is 403 or 404 and silenced.
             aiohttp.ClientResponseError: If the response status is 400 or higher and not silenced.
-
         """
         silenced = response.status in self.silence_errors_for_response_status_codes
 
@@ -77,7 +81,6 @@ async def _run(self, extra_data=None):
 
         This method provides the same return object type and documented in
         :meth:`~pulpcore.plugin.download.BaseDownloader._run`.
-
         """
         if self.sles_auth_token:
             auth_param = f"?{self.sles_auth_token}"
@@ -94,3 +97,110 @@ async def _run(self, extra_data=None):
         if self._close_session_on_finalize:
             self.session.close()
         return to_return
+
+
+class UlnDownloader(RpmDownloader):
+    """
+    Custom Downloader for ULN repositories.
+
+    Args:
+        username (str): Username for authentication in ULN network
+        password (str): password for authentication in ULN network
+        uln_server_base_url (str): ULN server url.
+
+    Raises:
+        UlnCredentialsError: If no or not valid ULN credentials are given,
+            this Error will be displayed
+    """
+
+    def __init__(self, *args, username=None, password=None, uln_server_base_url=None, **kwargs):
+        """
+        Initialize the downloader for ULN repositories.
+
+        If no server URL is given, use the server url from oracle linux.
+        """
+        self.username = username
+        self.password = password
+        self.uln_server_base_url = uln_server_base_url
+        self.headers = None
+        self.session_key = None
+
+        super().__init__(*args, **kwargs)
+
+    async def _run(self, extra_data=None):
+        """
+        Download, validate, and compute digests on the `url`. This is a coroutine.
+
+        Once per session the coroutine logs into the ULN account using the
+        ULN username and password. The returned key is used for authentification
+        for all other downloads.
+
+        This method provides the same return object type and documented in
+        :meth:`~pulpcore.plugin.download.BaseDownloader._run`.
+        """
+        parsed = urlparse(self.url)
+        if parsed.scheme == "uln":
+            # get ULN Session-key
+            SERVER_URL = os.path.join(self.uln_server_base_url, "rpc/api")
+
+            # set proxy for authentification
+            client = AllowProxyServerProxy(SERVER_URL, proxy=self.proxy, auth=self.auth)
+            if not self.session_key:
+                self.session_key = await client["auth.login"](self.username, self.password)
+                if len(self.session_key) != 43:
+                    raise UlnCredentialsError("No valid ULN credentials given.")
+                self.headers = {"X-ULN-API-User-Key": self.session_key}
+                await client.close()
+            # build request url from input uri
+            channelLabel = parsed.netloc
+            path = parsed.path.lstrip("/")
+            url = os.path.join(self.uln_server_base_url, "XMLRPC/GET-REQ", channelLabel, path)
+
+        async with self.session.get(
+            url, proxy=self.proxy, auth=self.auth, headers=self.headers
+        ) as response:
+            self.raise_for_status(response)
+            to_return = await self._handle_response(response)
+            await response.release()
+            self.response_headers = response.headers
+
+        if self._close_session_on_finalize:
+            self.session.close()
+            client.close()
+        return to_return
+
+
+class AllowProxyServerProxy(ServerProxy):
+    """
+    Overwriting the class aiohttp_xmlrpc.ServreProxy to allow proxy handling.
+
+    Until aiohttp-xmlrpc allows http post with proxy, use this patch.
+    This only works for http connection to the proxy, https connections are not supported!
+    """
+
+    def __init__(self, *args, proxy, auth, **kwargs):
+        """
+        Initialisation with proxy.
+        """
+        self.proxy = proxy
+        self.auth = auth
+        super().__init__(*args, **kwargs)
+
+    async def __remote_call(self, method_name, *args, **kwargs):
+        """
+        Set proxy for HTTP POST call.
+        """
+        async with self.client.post(
+            str(self.url),
+            data=etree.tostring(
+                self._make_request(method_name, *args, **kwargs),
+                xml_declaration=True,
+                encoding=self.encoding,
+            ),
+            headers=self.headers,
+            proxy=self.proxy,
+            auth=self.auth,
+        ) as response:
+            response.raise_for_status()
+
+            return self._parse_response((await response.read()), method_name)

pulp_rpm/app/exceptions.py

@@ -43,3 +43,25 @@ def __str__(self):
         Return a message for the exception.
         """
         return self.msg
+
+
+class UlnCredentialsError(PulpException):
+    """
+    Raised when no valid ULN Credentials were given.
+    """
+
+    def __init__(self, msg):
+        """
+        Set the exception identifier.
+
+        Args:
+            msg(str): Detailed message about the reasons for Distribution Tree conflict
+        """
+        super().__init__("RPM0003")
+        self.msg = msg
+
+    def __str__(self):
+        """
+        Return a message for the exception.
+        """
+        return self.msg

pulp_rpm/app/migrations/0031_ulnremote.py

@@ -0,0 +1,26 @@
+# Generated by Django 2.2.19 on 2021-03-18 16:30
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0061_call_handle_artifact_checksums_command'),
+        ('rpm', '0030_DATA_fix_updaterecord'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='UlnRemote',
+            fields=[
+                ('remote_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='rpm_ulnremote', serialize=False, to='core.Remote')),
+                ('uln_server_base_url', models.CharField(max_length=512, null=True)),
+            ],
+            options={
+                'default_related_name': '%(app_label)s_%(model_name)s',
+            },
+            bases=('core.remote',),
+        ),
+    ]

pulp_rpm/app/models/__init__.py

@@ -9,4 +9,4 @@
 from .distribution import Addon, Checksum, DistributionTree, Image, Variant  # noqa
 from .modulemd import Modulemd, ModulemdDefaults  # noqa
 from .package import Package  # noqa
-from .repository import RpmDistribution, RpmPublication, RpmRemote, RpmRepository  # noqa
+from .repository import RpmDistribution, RpmPublication, RpmRemote, UlnRemote, RpmRepository  # noqa

pulp_rpm/app/models/repository.py

@@ -37,7 +37,7 @@
     UpdateRecord,
 )
 
-from pulp_rpm.app.downloaders import RpmDownloader, RpmFileDownloader
+from pulp_rpm.app.downloaders import RpmDownloader, RpmFileDownloader, UlnDownloader
 from pulp_rpm.app.exceptions import DistributionTreeConflict
 from pulp_rpm.app.shared_utils import urlpath_sanitize
 
@@ -103,6 +103,74 @@ class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
 
 
+class UlnRemote(Remote):
+    """
+    Remote for "uln" content.
+    """
+
+    TYPE = "uln"
+    uln_server_base_url = models.CharField(max_length=512, null=True)
+
+    @property
+    def download_factory(self):
+        """
+        Return the DownloaderFactory which can be used to generate asyncio capable downloaders.
+
+        Returns:
+            DownloadFactory: The instantiated DownloaderFactory to be used by
+                get_downloader()
+
+        """
+        try:
+            return self._download_factory
+        except AttributeError:
+            self._download_factory = DownloaderFactory(
+                self,
+                downloader_overrides={
+                    "uln": UlnDownloader,
+                },
+            )
+            self._download_factory._handler_map["uln"] = self._download_factory._http_or_https
+            return self._download_factory
+
+    def get_downloader(self, remote_artifact=None, url=None, **kwargs):
+        """
+        Get a downloader from either a RemoteArtifact or URL that is configured with this Remote.
+
+        This method accepts either `remote_artifact` or `url` but not both. At least one is
+        required. If neither or both are passed a ValueError is raised.
+
+        Args:
+            remote_artifact (:class:`~pulpcore.app.models.RemoteArtifact`): The RemoteArtifact to
+                download.
+            url (str): The URL to download. Can be a ULN url.
+            kwargs (dict): This accepts the parameters of
+                :class:`~pulpcore.plugin.download.BaseDownloader`.
+        Raises:
+            ValueError: If neither remote_artifact and url are passed, or if both are passed.
+        Returns:
+            subclass of :class:`~pulpcore.plugin.download.BaseDownloader`: A downloader that
+            is configured with the remote settings.
+
+        """
+        if self.uln_server_base_url:
+            uln_server_base_url = self.uln_server_base_url
+        else:
+            uln_server_base_url = settings.DEFAULT_ULN_SERVER_BASE_URL
+
+        return super().get_downloader(
+            remote_artifact=remote_artifact,
+            url=url,
+            username=self.username,
+            password=self.password,
+            uln_server_base_url=uln_server_base_url,
+            **kwargs,
+        )
+
+    class Meta:
+        default_related_name = "%(app_label)s_%(model_name)s"
+
+
 class RpmRepository(Repository):
     """
     Repository for "rpm" content.

pulp_rpm/app/serializers/__init__.py

@@ -24,6 +24,7 @@
     RpmDistributionSerializer,
     RpmPublicationSerializer,
     RpmRemoteSerializer,
+    UlnRemoteSerializer,
     RpmRepositorySerializer,
     RpmRepositorySyncURLSerializer,
 )

pulp_rpm/app/serializers/repository.py

@@ -29,6 +29,7 @@
     RpmRemote,
     RpmRepository,
     RpmPublication,
+    UlnRemote,
 )
 from pulp_rpm.app.schema import COPY_CONFIG_SCHEMA
 
@@ -64,17 +65,11 @@ class Meta:
         model = RpmRepository
 
 
-class RpmRemoteSerializer(RemoteSerializer):
+class RpmBaseRemoteSerializer(RemoteSerializer):
     """
-    A Serializer for RpmRemote.
+    A common base serializer for multiple RPM based remotes.
     """
 
-    sles_auth_token = serializers.CharField(
-        help_text=_("Authentication token for SLES repositories."),
-        required=False,
-        allow_null=True,
-    )
-
     policy = serializers.ChoiceField(
         help_text=_(
             "The policy to use when downloading content. The possible values include: "
@@ -84,11 +79,60 @@ class RpmRemoteSerializer(RemoteSerializer):
         default=Remote.IMMEDIATE,
     )
 
+
+class RpmRemoteSerializer(RpmBaseRemoteSerializer):
+    """
+    A Serializer for RpmRemote.
+    """
+
+    sles_auth_token = serializers.CharField(
+        help_text=_("Authentication token for SLES repositories."),
+        required=False,
+        allow_null=True,
+    )
+
     class Meta:
         fields = RemoteSerializer.Meta.fields + ("sles_auth_token",)
         model = RpmRemote
 
 
+class UlnRemoteSerializer(RpmBaseRemoteSerializer):
+    """
+    A Serializer for UlnRemote.
+    """
+
+    username = serializers.CharField(
+        help_text=_("Your ULN account username."),
+        required=True,
+    )
+
+    password = serializers.CharField(
+        help_text=_("Your ULN account password."),
+        required=True,
+    )
+
+    url = serializers.CharField(
+        help_text=_(
+            "The ULN repo URL of the remote content source."
+            '"This is "uln://" followed by the channel name. E.g.: "uln://ol7_x86_64_oracle"'
+        ),
+        required=True,
+    )
+
+    uln_server_base_url = serializers.CharField(
+        help_text=_(
+            "Base URL of the ULN server. If the uln_server_base_url is not provided pulp_rpm will"
+            "use the contents of the DEFAULT_ULN_SERVER_BASE_URL setting instead."
+        ),
+        required=False,
+        allow_null=True,
+    )
+
+    class Meta:
+        fields = RemoteSerializer.Meta.fields + ("uln_server_base_url",)
+        model = UlnRemote
+
+
 class RpmPublicationSerializer(PublicationSerializer):
     """
     A Serializer for RpmPublication.