Fixes ability to specify custom headers on a Remote

The Remotes APIs accepts a list of headers to be sent with each request during
sync. However, the headers were not being properly passed to aiohttp. This patch
fixes how the headers are passed to aiohttp.

This patch also ensures that any User-Agent headers configured on a remote are
appended to the User-Agent string generated by pulp.

fixes: #8689
https://pulp.plan.io/issues/8689

CHANGES/8689.bugfix

@@ -0,0 +1 @@
+Fixed ability to specify custom headers on a Remote.

pulpcore/download/factory.py

@@ -3,6 +3,7 @@
 import atexit
 import copy
 from gettext import gettext as _
+from multidict import MultiDict
 import platform
 from pkg_resources import get_distribution
 import ssl
@@ -15,8 +16,6 @@
 from .http import HttpDownloader
 from .file import FileDownloader
 
-import json
-
 
 PROTOCOL_MAP = {
     "http": HttpDownloader,
@@ -124,9 +123,13 @@ def _make_aiohttp_session_from_remote(self):
         if sslcontext:
             tcp_conn_opts["ssl_context"] = sslcontext
 
-        headers = {"User-Agent": user_agent()}
+        headers = MultiDict({"User-Agent": user_agent()})
         if self._remote.headers is not None:
-            headers.update(json.loads(self._remote.headers))
+            for header_dict in self._remote.headers:
+                user_agent_header = header_dict.pop("User-Agent", None)
+                if user_agent_header:
+                    headers["User-Agent"] = f"{headers['User-Agent']}, {user_agent_header}"
+                headers.extend(header_dict)
 
         conn = aiohttp.TCPConnector(**tcp_conn_opts)
         total = self._remote.total_timeout

pulpcore/tests/functional/api/using_plugin/test_crud_repos.py

@@ -346,3 +346,14 @@ def test_delete(self):
         # verify the delete
         with self.assertRaises(ApiException):
             self.remotes_api.read(self.remote.pulp_href)
+
+    def test_headers(self):
+        # Test that headers value must be a list of dicts
+        data = {"headers": {"Connection": "keep-alive"}}
+        with self.assertRaises(ApiException):
+            self.remotes_api.partial_update(self.remote.pulp_href, data)
+        data = {"headers": [1, 2, 3]}
+        with self.assertRaises(ApiException):
+            self.remotes_api.partial_update(self.remote.pulp_href, data)
+        data = {"headers": [{"Connection": "keep-alive"}]}
+        self.remotes_api.partial_update(self.remote.pulp_href, data)

pulpcore/tests/functional/api/using_plugin/test_repo_versions.py

@@ -133,6 +133,7 @@ def test_02_sync_content(self):
         * The ``content_removed_summary`` attribute is correct.
         """
         body = gen_file_remote()
+        body.update({"headers": [{"Connection": "keep-alive"}]})
         self.remote.update(self.client.post(FILE_REMOTE_PATH, body))
         sync(self.cfg, self.remote, self.repo)
         repo = self.client.get(self.repo["pulp_href"])

pulpcore/tests/unit/download/test_factory.py

@@ -0,0 +1,33 @@
+from django.test import TestCase
+
+from pulpcore.download.factory import DownloaderFactory, user_agent
+from pulpcore.plugin.models import Remote
+
+
+class DownloaderFactoryHeadersTestCase(TestCase):
+    def test_user_agent_header(self):
+        remote = Remote.objects.create(url="http://example.org/", name="foo")
+        factory = DownloaderFactory(remote)
+        downloader = factory.build(remote.url)
+        default_user_agent = user_agent()
+        self.assertEqual(downloader.session.headers["User-Agent"], default_user_agent)
+        remote.delete()
+
+    def test_custom_user_agent_header(self):
+        remote = Remote.objects.create(
+            url="http://example.org/", headers=[{"User-Agent": "foo"}], name="foo"
+        )
+        factory = DownloaderFactory(remote)
+        downloader = factory.build(remote.url)
+        default_user_agent = user_agent()
+        expected_user_agent = f"{default_user_agent}, foo"
+        self.assertEqual(downloader.session.headers["User-Agent"], expected_user_agent)
+        remote.delete()
+
+    def test_custom_headers(self):
+        remote = Remote.objects.create(
+            url="http://example.org/", headers=[{"Connection": "keep-alive"}], name="foo"
+        )
+        factory = DownloaderFactory(remote)
+        downloader = factory.build(remote.url)
+        self.assertEqual(downloader.session.headers["Connection"], "keep-alive")