Verify Release signatures in sync

fixes #6170
https://pulp.plan.io/issues/6170

CHANGES/6170.feature

@@ -0,0 +1 @@
+Verification of upstream signed metadata has been implemented.

pulp_deb/app/migrations/0008_debremote_gpgkey.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.10 on 2020-02-17 12:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('deb', '0007_create_metadata_models'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='debremote',
+            name='gpgkey',
+            field=models.TextField(null=True),
+        ),
+    ]

pulp_deb/app/models/remote.py

@@ -20,6 +20,7 @@ class DebRemote(Remote):
     sync_sources = models.BooleanField(default=False)
     sync_udebs = models.BooleanField(default=False)
     sync_installer = models.BooleanField(default=False)
+    gpgkey = models.TextField(null=True)
 
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"

pulp_deb/app/serializers/remote_serializers.py

@@ -12,15 +12,15 @@ class DebRemoteSerializer(RemoteSerializer):
     """
 
     distributions = CharField(
-        help_text="Whitespace separated list of distributions to sync", required=True
+        help_text="Whitespace separated list of distributions to sync", required=True,
     )
 
     components = CharField(
-        help_text="Whitespace separatet list of components to sync", required=False
+        help_text="Whitespace separatet list of components to sync", required=False,
     )
 
     architectures = CharField(
-        help_text="Whitespace separated list of architectures to sync", required=False
+        help_text="Whitespace separated list of architectures to sync", required=False,
     )
 
     sync_sources = BooleanField(help_text="Sync source packages", required=False)
@@ -29,6 +29,10 @@ class DebRemoteSerializer(RemoteSerializer):
 
     sync_installer = BooleanField(help_text="Sync installer files", required=False)
 
+    gpgkey = CharField(
+        help_text="Gpg public key to verify origin releases against", required=False,
+    )
+
     policy = ChoiceField(
         help_text="The policy to use when downloading content. The possible values include: "
         "'immediate', 'on_demand', and 'streamed'. 'immediate' is the default.",
@@ -44,5 +48,6 @@ class Meta:
             "sync_sources",
             "sync_udebs",
             "sync_installer",
+            "gpgkey",
         )
         model = DebRemote

pulp_deb/app/tasks/synchronizing.py

@@ -7,6 +7,7 @@
 import bz2
 import gzip
 import lzma
+import gnupg
 from collections import defaultdict
 from tempfile import NamedTemporaryFile
 
@@ -50,6 +51,18 @@
 log = logging.getLogger(__name__)
 
 
+class NoReleaseFile(Exception):
+    """
+    Exception to signal, that no file representing a release is present.
+    """
+
+    def __init__(self, distribution, *args, **kwargs):
+        """
+        Exception to signal, that no file representing a release is present.
+        """
+        super().__init__("No valid Release file found for {}".format(distribution), *args, **kwargs)
+
+
 class NoPackageIndexFile(Exception):
     """
     Exception to signal, that no file representing a package index is present.
@@ -126,9 +139,9 @@ def pipeline_stages(self, new_version):
             self.first_stage,
             QueryExistingArtifacts(),
             ArtifactDownloader(),
-            DebDropEmptyContent(),
+            DebDropFailedArtifacts(),
             ArtifactSaver(),
-            DebUpdateReleaseFileAttributes(),
+            DebUpdateReleaseFileAttributes(remote=self.first_stage.remote),
             DebUpdatePackageIndexAttributes(),
             QueryExistingContents(),
             ContentSaver(),
@@ -155,6 +168,20 @@ class DebUpdateReleaseFileAttributes(Stage):
     TODO: Verify signature
     """
 
+    def __init__(self, remote, *args, **kwargs):
+        """Initialize DebUpdateReleaseFileAttributes stage."""
+        super().__init__(*args, **kwargs)
+        self.remote = remote
+        self.gpgkey = remote.gpgkey
+        if self.gpgkey:
+            gnupghome = os.path.join(os.getcwd(), "gpg-home")
+            os.makedirs(gnupghome)
+            self.gpg = gnupg.GPG(gpgbinary="/usr/bin/gpg", gnupghome=gnupghome)
+            import_res = self.gpg.import_keys(self.gpgkey)
+            if import_res.count == 0:
+                log.warn("Key import failed.")
+            pass
+
     async def run(self):
         """
         Parse ReleaseFile content units.
@@ -168,17 +195,55 @@ async def run(self):
                     da_names = {
                         os.path.basename(da.relative_path): da for da in d_content.d_artifacts
                     }
-                    if "InRelease" in da_names:
-                        release_file_artifact = da_names["InRelease"].artifact
-                        release_file.relative_path = da_names["InRelease"].relative_path
-                    elif "Release" in da_names:
-                        release_file_artifact = da_names["Release"].artifact
-                        release_file.relative_path = da_names["Release"].relative_path
+                    if "Release" in da_names:
+                        if "Release.gpg" in da_names:
+                            if self.gpgkey:
+                                with NamedTemporaryFile() as tmp_file:
+                                    tmp_file.write(da_names["Release"].artifact.file.read())
+                                    tmp_file.flush()
+                                    verified = self.gpg.verify_file(
+                                        da_names["Release.gpg"].artifact.file, tmp_file.name
+                                    )
+                                if verified.valid:
+                                    log.info("Verification of Release successful.")
+                                    release_file_artifact = da_names["Release"].artifact
+                                    release_file.relative_path = da_names["Release"].relative_path
+                                else:
+                                    log.warn("Verification of Release failed. Dropping it.")
+                                    d_content.d_artifacts.remove(da_names.pop("Release"))
+                                    d_content.d_artifacts.remove(da_names.pop("Release.gpg"))
+                            else:
+                                release_file_artifact = da_names["Release"].artifact
+                                release_file.relative_path = da_names["Release"].relative_path
+                        else:
+                            if self.gpgkey:
+                                d_content.d_artifacts.delete(da_names["Release"])
+                            else:
+                                release_file_artifact = da_names["Release"].artifact
+                                release_file.relative_path = da_names["Release"].relative_path
                     else:
-                        # No (proper) artifacts left -> drop it
-                        d_content.content = None
-                        d_content.resolve()
-                        continue
+                        if "Release.gpg" in da_names:
+                            # No need to keep the signature without "Release"
+                            d_content.d_artifacts.remove(da_names.pop("Release.gpg"))
+
+                    if "InRelease" in da_names:
+                        if self.gpgkey:
+                            verified = self.gpg.verify_file(da_names["InRelease"].artifact.file)
+                            if verified.valid:
+                                log.info("Verification of InRelease successful.")
+                                release_file_artifact = da_names["InRelease"].artifact
+                                release_file.relative_path = da_names["InRelease"].relative_path
+                            else:
+                                log.warn("Verification of InRelease failed. Dropping it.")
+                                d_content.d_artifacts.remove(da_names.pop("InRelease"))
+                        else:
+                            release_file_artifact = da_names["InRelease"].artifact
+                            release_file.relative_path = da_names["InRelease"].relative_path
+
+                    if not d_content.d_artifacts:
+                        # No (proper) artifacts left -> distribution not found
+                        raise NoReleaseFile(distribution=release_file.distribution)
+
                     release_file.sha256 = release_file_artifact.sha256
                     release_file_dict = deb822.Release(release_file_artifact.file)
                     release_file.codename = release_file_dict["Codename"]
@@ -257,27 +322,21 @@ def _uncompress_artifact(d_artifacts):
     raise NoPackageIndexFile()
 
 
-class DebDropEmptyContent(Stage):
+class DebDropFailedArtifacts(Stage):
     """
-    This stage removes empty DeclarativeContent objects.
+    This stage removes failed failsafe artifacts.
 
     In case we tried to fetch something, but the artifact 404ed, we simply drop it.
     """
 
     async def run(self):
         """
-        Drop GenericContent units if they have no artifacts left.
+        Remove None from d_artifacts in DeclarativeContent units.
         """
         async for d_content in self.items():
-            if d_content.d_artifacts:  # Should there be artifacts?
-                d_content.d_artifacts = [
-                    d_artifact for d_artifact in d_content.d_artifacts if d_artifact.artifact
-                ]
-                if not d_content.d_artifacts:
-                    # No artifacts left -> drop it
-                    d_content.content = None
-                    d_content.resolve()
-                    continue
+            d_content.d_artifacts = [
+                d_artifact for d_artifact in d_content.d_artifacts if d_artifact.artifact
+            ]
             await self.put(d_content)
 
 

pulp_deb/tests/functional/api/test_crud_remotes.py

@@ -5,7 +5,7 @@
 
 from pulp_smash import utils
 
-from pulp_deb.tests.functional.constants import DOWNLOAD_POLICIES
+from pulp_deb.tests.functional.constants import DOWNLOAD_POLICIES, DEB_SIGNING_KEY
 from pulp_deb.tests.functional.utils import (
     deb_remote_api,
     gen_deb_remote,
@@ -216,6 +216,7 @@ def _gen_verbose_remote():
             "distributions": "{} {}".format(utils.uuid4(), utils.uuid4()),
             "components": "{} {}".format(utils.uuid4(), utils.uuid4()),
             "architectures": "{} {}".format(utils.uuid4(), utils.uuid4()),
+            "gpgkey": DEB_SIGNING_KEY,
         }
     )
     return attrs

pulp_deb/tests/functional/api/test_sync.py

@@ -10,6 +10,9 @@
     DEB_FIXTURE_SUMMARY,
     DEB_FULL_FIXTURE_SUMMARY,
     DEB_INVALID_FIXTURE_URL,
+    DEB_FIXTURE_URL,
+    DEB_FIXTURE_RELEASE,
+    DEB_SIGNING_KEY,
 )
 from pulp_deb.tests.functional.utils import set_up_module as setUpModule  # noqa:F401
 from pulp_deb.tests.functional.utils import (
@@ -65,7 +68,7 @@ def do_sync(self, sync_udebs, fixture_summary):
         repo = repo_api.create(gen_repo())
         self.addCleanup(repo_api.delete, repo.pulp_href)
 
-        body = gen_deb_remote(sync_udebs=sync_udebs)
+        body = gen_deb_remote(sync_udebs=sync_udebs, gpgkey=DEB_SIGNING_KEY)
         remote = remote_api.create(body)
         self.addCleanup(remote_api.delete, remote.pulp_href)
 
@@ -137,9 +140,19 @@ def test_invalid_url(self):
         Test that we get a task failure. See :meth:`do_test`.
         """
         with self.assertRaises(PulpTaskError) as exc:
-            self.do_test("http://i-am-an-invalid-url.com/invalid/")
+            self.do_test(url="http://i-am-an-invalid-url.com/invalid/")
         error = exc.exception.task.error
-        self.assertIsNotNone(error["description"])
+        self.assertIn("Cannot connect", error["description"])
+
+    def test_invalid_distribution(self):
+        """Sync a repository using a distribution that does not exist.
+
+        Test that we get a task failure. See :meth:`do_test`.
+        """
+        with self.assertRaises(PulpTaskError) as exc:
+            self.do_test(distribution="no_dist")
+        error = exc.exception.task.error
+        self.assertIn("No valid Release file found", error["description"])
 
     # Provide an invalid repository and specify keywords in the anticipated error message
     @unittest.skip("FIXME: Plugin writer action required.")
@@ -150,20 +163,20 @@ def test_invalid_deb_content(self):
         keywords related to the reason of the failure. See :meth:`do_test`.
         """
         with self.assertRaises(PulpTaskError) as exc:
-            self.do_test(DEB_INVALID_FIXTURE_URL)
+            self.do_test(url=DEB_INVALID_FIXTURE_URL)
         error = exc.exception.task.error
         for key in ("mismached", "empty"):
             self.assertIn(key, error["description"])
 
-    def do_test(self, url):
+    def do_test(self, url=DEB_FIXTURE_URL, distribution=DEB_FIXTURE_RELEASE):
         """Sync a repository given ``url`` on the remote."""
         repo_api = deb_repository_api
         remote_api = deb_remote_api
 
         repo = repo_api.create(gen_repo())
         self.addCleanup(repo_api.delete, repo.pulp_href)
 
-        body = gen_deb_remote(url=url)
+        body = gen_deb_remote(url=url, distributions=distribution)
         remote = remote_api.create(body)
         self.addCleanup(remote_api.delete, remote.pulp_href)
 

pulp_deb/tests/functional/constants.py

@@ -98,3 +98,34 @@ def _clean_dict(d):
 
 # FIXME: replace this with the actual number of content units in your test fixture
 DEB_LARGE_FIXTURE_COUNT = 25
+
+DEB_SIGNING_KEY = """-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v2
+
+mQENBFek0GkBCACwGSRiUSE3d+0vA7/X7xj+6u4y5Pg43G6AZIeUrNN4+Y7z2s/y
+VWBWfjimJevQUBbOn5Otm/9wBNAcTKAMEqlVGmsRPKonPT3SHeX9dVo2LkbOZJDR
+kdEu1TX6wiuuhZAsJoPM0cClF2IV9xSQN3o4xW8oo63/ZLRu3lCraia0sfob3jZi
+cYUI9cC6OOLmH+1nmcCVo1qg3zSZg/gFyvscVMr1Dm5PfjyH/1SO4MgK6RqHkrxV
+dhvwBPs1bO9dzjB7H1Lmyb2l0lFOrArqPW3jgcKV1+AmpJGshLyOQBmZ2rW7oGTG
+il33iSSrZ4TKjj6y3392gxX3gs4bYvB8hjotABEBAAG0B1B1bHAgUUWJATcEEwEI
+ACEFAlek0GkCGwMFCwkIBwIGFQgJCgsCBBYCAwECHgECF4AACgkQBaXm2iadnZhP
+vAf9GG8foj1EaBTENXgH+7Zc1aKur7i738felcqhZhUlZBD8vyPrh0TPJ63uITXS
+9RiE70/iwsDqKY8RiB8oMENI2CAEHXEelLC7Qx5f97WVaNmlydOQBxs4V09T8pDg
+BK21D3/HLBL0QjW6uE7TAEGuiCd8A1ZKvjNyQhCtElDKjgOT2LtvlH6L3PZ+KWnA
+l4n7wSADkgyU+n+jGorKH4yxxVHelnpNNas5AhI/cB73i9lhR8iQL6RDKgQuc0fy
+wW4gsAoxjH2SeCJgxRIF2ezrCedS1chgnQAvItKmHsuLJHuWNT0QuG5nLzNVjjh1
+L8YUJiVGxxqzEIQ/HrQdZBPlL7kBDQRXpNBpAQgAu9+1oHg1uhCQTwjMRNpPT6qr
+z8gvVepfUK7UzHvtBjRMVcUmfHVOwURNpd6qPNu7tsGe/KuvrMFU9pwmq+zIytX4
+vmY8BBtIIoHTeC3DtoWrpemXZht5jDL8kgygCNqGg9E6TvdqDZ6ItDOAP3wBkieR
+LghwPG3KylQFudRJq5qbWpzrX2RRIVSSiLLl/zttiYKE1eCimUQ12nztey/eN+VV
+u5U+y88xJr40vnPEkPKQmE713xuzIUAFXEx6FxDUMWNPUPPJtlfIe9QsjxNZ4R9w
+s/arq5TILiiqmOHpnu8gcEjfDu8n10AKMUgdc2NocqmetyPnb8KZon1oX+IN0wAR
+AQABiQEfBBgBCAAJBQJXpNBpAhsMAAoJEAWl5tomnZ2YP2MIAJtzsoRbzLtixNWP
+PoYXPW5eUZ/R+9pV6agAZYwzTmuCNRzTV2vxgCGvnvzC0SZbvBKeVqONBuTariyo
+aC4Y1pUj5xX6AOIt0gbyMsj+XcYz2SuRuB+fAW1avmBaBI7jlsqHkPGBqTdeVbJC
+qKhCv0igH3jv/222eWEp5w7V7Xre1IyNCtyn8qeN9igH+5XyPmiV04PndmORusFq
+CeEE45C7ahpX9VJ8fwZ+XJBRYxoaRJ1tpAVrNeJsXxiGxJGmuL86hdJN/1W1G8QT
+gAMUtmcqiACuLWVpljMJKzuVaIqXq9nNMRTUzGFIG0dSmA6pNeym9RFPW2ro3G11
+uUBsbCg=
+=8Is3
+-----END PGP PUBLIC KEY BLOCK-----"""

pulp_deb/tests/functional/utils.py

@@ -88,12 +88,14 @@ def gen_deb_client():
 
 
 def gen_deb_remote(
-    url=DEB_FIXTURE_URL, distributions=DEB_FIXTURE_RELEASE, sync_udebs=False, **kwargs
+    url=DEB_FIXTURE_URL, distributions=DEB_FIXTURE_RELEASE, sync_udebs=False, gpgkey=None, **kwargs
 ):
     """Return a semi-random dict for use in creating a deb Remote.
 
     :param url: The URL of an external content source.
     """
+    if gpgkey:
+        kwargs["gpgkey"] = gpgkey
     return gen_remote(url, distributions=distributions, sync_udebs=sync_udebs, **kwargs)