Add Release signing to publishing

Required PR: pulp/pulpcore#558

fixes #6171

.travis/post_before_script.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+set -euv
+
+if [ "$TEST" = "pulp" ] || [ "$TEST" = "performance" ]
+then
+  # Aliases for running commands in the pulp-worker container.
+  export PULP_WORKER_POD=$(sudo kubectl get pods | grep -E -o "pulp-worker-(\w+)-(\w+)")
+  # Run a command
+  export CMD_WORKER_PREFIX="sudo kubectl exec $PULP_WORKER_POD --"
+  # Run a command, and pass STDIN
+  export CMD_WORKER_STDIN_PREFIX="sudo kubectl exec -i $PULP_WORKER_POD --"
+
+  cat $TRAVIS_BUILD_DIR/pulp_deb/tests/functional/sign_deb_release.sh | $CMD_WORKER_STDIN_PREFIX bash -c "cat > /root/sign_deb_release.sh"
+  cat $TRAVIS_BUILD_DIR/pulp_deb/tests/functional/setup_signing_service.py | $CMD_WORKER_STDIN_PREFIX bash -c "cat > /tmp/setup_signing_service.py"
+
+  curl -L https://github.com/pulp/pulp-fixtures/raw/master/common/GPG-PRIVATE-KEY-pulp-qe | $CMD_WORKER_STDIN_PREFIX gpg --import
+  echo "6EDF301256480B9B801EBA3D05A5E6DA269D9D98:6:" | $CMD_WORKER_STDIN_PREFIX gpg --import-ownertrust
+  $CMD_WORKER_PREFIX chmod a+x /tmp/setup_signing_service.py /root/sign_deb_release.sh
+  $CMD_WORKER_PREFIX /tmp/setup_signing_service.py /root/sign_deb_release.sh
+fi

CHANGES/6171.feature

@@ -0,0 +1 @@
+Add support for signing release files

pulp_deb/app/migrations/0009_debpublication_signing_service.py

@@ -0,0 +1,20 @@
+# Generated by Django 2.2.10 on 2020-02-21 09:24
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0021_add_signing_service_foreign_key'),
+        ('deb', '0008_debremote_gpgkey'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='debpublication',
+            name='signing_service',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='deb_debpublication', to='core.SigningService'),
+        ),
+    ]

pulp_deb/app/models/publication.py

@@ -2,7 +2,7 @@
 
 from django.db import models
 
-from pulpcore.plugin.models import Publication, PublicationDistribution
+from pulpcore.plugin.models import Publication, PublicationDistribution, SigningService
 
 logger = getLogger(__name__)
 
@@ -33,6 +33,7 @@ class DebPublication(Publication):
 
     simple = models.BooleanField(default=False)
     structured = models.BooleanField(default=False)
+    signing_service = models.ForeignKey(SigningService, on_delete=models.PROTECT, null=True)
 
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"

pulp_deb/app/serializers/publication_serializers.py

@@ -1,6 +1,7 @@
-from rest_framework.serializers import BooleanField, ValidationError
+from rest_framework.serializers import BooleanField, ValidationError, HyperlinkedRelatedField
 from pulpcore.plugin.serializers import PublicationDistributionSerializer, PublicationSerializer
 
+from pulpcore.plugin.models import AsciiArmoredDetachedSigningService
 from pulp_deb.app.models import DebDistribution, DebPublication, VerbatimPublication
 
 
@@ -24,6 +25,13 @@ class DebPublicationSerializer(PublicationSerializer):
         default=False,
     )
     structured = BooleanField(help_text="Activate structured publishing mode.", default=False)
+    signing_service = HyperlinkedRelatedField(
+        help_text="Sign Release files with this signing key",
+        many=False,
+        queryset=AsciiArmoredDetachedSigningService.objects.all(),
+        view_name="signing-services-detail",
+        required=False,
+    )
 
     def validate(self, data):
         """
@@ -35,7 +43,7 @@ def validate(self, data):
         return data
 
     class Meta:
-        fields = PublicationSerializer.Meta.fields + ("simple", "structured")
+        fields = PublicationSerializer.Meta.fields + ("simple", "structured", "signing_service")
         model = DebPublication
 
 

pulp_deb/app/tasks/publishing.py

@@ -8,7 +8,12 @@
 
 from django.core.files import File
 
-from pulpcore.plugin.models import PublishedArtifact, PublishedMetadata, RepositoryVersion
+from pulpcore.plugin.models import (
+    PublishedArtifact,
+    PublishedMetadata,
+    RepositoryVersion,
+    AsciiArmoredDetachedSigningService,
+)
 from pulpcore.plugin.tasking import WorkingDirectory
 
 from pulp_deb.app.models import (
@@ -48,18 +53,22 @@ def publish_verbatim(repository_version_pk):
     log.info(_("Publication (verbatim): {publication} created").format(publication=publication.pk))
 
 
-def publish(repository_version_pk, simple=False, structured=False):
+def publish(repository_version_pk, simple=False, structured=False, signing_service_pk=None):
     """
     Use provided publisher to create a Publication based on a RepositoryVersion.
 
     Args:
         repository_version_pk (str): Create a publication from this repository version.
         simple (bool): Create a simple publication with all packages contained in default/all.
         structured (bool): Create a structured publication with releases and components.
-            (Not yet implemented)
+        signing_service_pk (str): Use this SigningService to sign the Release files.
 
     """
     repo_version = RepositoryVersion.objects.get(pk=repository_version_pk)
+    if signing_service_pk:
+        signing_service = AsciiArmoredDetachedSigningService.objects.get(pk=signing_service_pk)
+    else:
+        signing_service = None
 
     log.info(
         _(
@@ -75,6 +84,7 @@ def publish(repository_version_pk, simple=False, structured=False):
         with DebPublication.create(repo_version, pass_through=False) as publication:
             publication.simple = simple
             publication.structured = structured
+            publication.signing_service = signing_service
             repository = repo_version.repository
 
             if simple:
@@ -195,6 +205,7 @@ def __init__(
         self.release["SHA512"] = []
         self.architectures = architectures
         self.components = {name: _ComponentHelper(self, name) for name in components}
+        self.signing_service = publication.signing_service
 
     def add_metadata(self, metadata):
         artifact = metadata._artifacts.get()
@@ -218,15 +229,27 @@ def finish(self):
             component.finish()
         # Publish Release file
         self.release["components"] = " ".join(self.components.keys())
-        release_path = os.path.join("dists", self.release["codename"], "Release")
+        release_dir = os.path.join("dists", self.release["codename"])
+        release_path = os.path.join(release_dir, "Release")
         os.makedirs(os.path.dirname(release_path), exist_ok=True)
         with open(release_path, "wb") as release_file:
             self.release.dump(release_file)
         release_metadata = PublishedMetadata.create_from_file(
             publication=self.publication, file=File(open(release_path, "rb")),
         )
         release_metadata.save()
-        # TODO Sign release
+        if self.signing_service:
+            in_release_path = os.path.join(release_dir, "InRelease")
+            release_gpg_path = os.path.join(release_dir, "Release.gpg")
+            self.signing_service.sign(release_path)
+            in_release_metadata = PublishedMetadata.create_from_file(
+                publication=self.publication, file=File(open(in_release_path, "rb")),
+            )
+            in_release_metadata.save()
+            release_gpg_metadata = PublishedMetadata.create_from_file(
+                publication=self.publication, file=File(open(release_gpg_path, "rb")),
+            )
+            release_gpg_metadata.save()
 
 
 def _zip_file(file_path):

pulp_deb/app/viewsets/publication.py

@@ -70,14 +70,16 @@ def create(self, request):
         repository_version = serializer.validated_data.get("repository_version")
         simple = serializer.validated_data.get("simple")
         structured = serializer.validated_data.get("structured")
+        signing_service = serializer.validated_data.get("signing_service")
 
         result = enqueue_with_reservation(
             tasks.publish,
             [repository_version.repository],
             kwargs={
-                "repository_version_pk": str(repository_version.pk),
+                "repository_version_pk": repository_version.pk,
                 "simple": simple,
                 "structured": structured,
+                "signing_service_pk": getattr(signing_service, "pk", None),
             },
         )
         return OperationPostponedResponse(result, request)

pulp_deb/tests/functional/api/test_publish.py

@@ -18,6 +18,7 @@
     deb_remote_api,
     deb_repository_api,
     deb_verbatim_publication_api,
+    signing_service_api,
 )
 
 from pulpcore.client.pulp_deb import (
@@ -39,11 +40,10 @@ class PublishAnyRepoVersionSimpleTestCase(unittest.TestCase):
 
     class Meta:
         publication_api = deb_apt_publication_api
+        Publication = DebDebPublication
 
-        @staticmethod
-        def Publication(*args, **kwargs):
-            """Delegate Publication constructor."""
-            return DebDebPublication(simple=True, *args, **kwargs)
+    def _publication_extra_args(self):
+        return {"simple": True}
 
     def test_all(self):
         """Test whether a particular repository version can be published.
@@ -84,7 +84,9 @@ def test_all(self):
         non_latest = choice(version_hrefs[:-1])
 
         # Step 2
-        publish_data = self.Meta.Publication(repository=repo.pulp_href)
+        publish_data = self.Meta.Publication(
+            repository=repo.pulp_href, **self._publication_extra_args()
+        )
         publish_response = publication_api.create(publish_data)
         created_resources = monitor_task(publish_response.task)
         publication_href = created_resources[0]
@@ -95,7 +97,9 @@ def test_all(self):
         self.assertEqual(publication.repository_version, version_hrefs[-1])
 
         # Step 4
-        publish_data = self.Meta.Publication(repository_version=non_latest)
+        publish_data = self.Meta.Publication(
+            repository_version=non_latest, **self._publication_extra_args()
+        )
         publish_response = publication_api.create(publish_data)
         created_resources = monitor_task(publish_response.task)
         publication_href = created_resources[0]
@@ -121,11 +125,10 @@ class PublishAnyRepoVersionStructuredTestCase(PublishAnyRepoVersionSimpleTestCas
 
     class Meta:
         publication_api = deb_apt_publication_api
+        Publication = DebDebPublication
 
-        @staticmethod
-        def Publication(*args, **kwargs):
-            """Delegate Publication constructor."""
-            return DebDebPublication(structured=True, *args, **kwargs)
+    def _publication_extra_args(self):
+        return {"structured": True}
 
 
 class PublishAnyRepoVersionCombinedTestCase(PublishAnyRepoVersionSimpleTestCase):
@@ -139,11 +142,40 @@ class PublishAnyRepoVersionCombinedTestCase(PublishAnyRepoVersionSimpleTestCase)
 
     class Meta:
         publication_api = deb_apt_publication_api
+        Publication = DebDebPublication
 
-        @staticmethod
-        def Publication(*args, **kwargs):
-            """Delegate Publication constructor."""
-            return DebDebPublication(simple=True, structured=True, *args, **kwargs)
+    def _publication_extra_args(self):
+        return {"simple": True, "structured": True}
+
+
+class PublishAnyRepoVersionSignedTestCase(PublishAnyRepoVersionSimpleTestCase):
+    """Test whether a particular repository version can be published with signed metadata.
+
+    This test targets the following issues:
+
+    * `PulpDeb #6171 <https://pulp.plan.io/issues/6171>`_
+    """
+
+    class Meta:
+        publication_api = deb_apt_publication_api
+        Publication = DebDebPublication
+
+    def _publication_extra_args(self):
+        return {
+            "simple": True,
+            "structured": True,
+            "signing_service": self.signing_service.pulp_href,
+        }
+
+    def setUp(self):
+        """Find SigningService for use in tests."""
+        response = signing_service_api.list(name="sign_deb_release")
+        if response.count == 0:
+            self.fail(
+                """No signing service setup.
+Please call pulp_deb/pulp_deb/tests/functional/setup_signing_service.py"""
+            )
+        self.signing_service = response.results[0]
 
 
 class VerbatimPublishAnyRepoVersionTestCase(PublishAnyRepoVersionSimpleTestCase):
@@ -158,3 +190,6 @@ class VerbatimPublishAnyRepoVersionTestCase(PublishAnyRepoVersionSimpleTestCase)
     class Meta:
         publication_api = deb_verbatim_publication_api
         Publication = DebVerbatimPublication
+
+    def _publication_extra_args(self):
+        return {}

pulp_deb/tests/functional/setup_signing_service.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: {} <path_to_signing_script>".format(sys.argv[0]))
+        sys.exit(1)
+
+    script_path = os.path.realpath(sys.argv[1])
+    if not os.path.exists(script_path):
+        print("Usage: {} <path_to_signing_script>".format(sys.argv[0]))
+        sys.exit(1)
+
+    import django
+
+    django.setup()
+
+    from pulpcore.app.models.content import AsciiArmoredDetachedSigningService
+
+    AsciiArmoredDetachedSigningService.objects.create(
+        name="sign_deb_release", script=script_path,
+    )

pulp_deb/tests/functional/sign_deb_release.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -e
+
+FILE_PATH="$(/usr/bin/readlink -f $1)"
+FILE_DIR="$(/usr/bin/dirname "${FILE_PATH}")"
+SIGNATURE_PATH="${FILE_DIR}/Release.gpg"
+INLINE_SIGNATURE_PATH="${FILE_DIR}/InRelease"
+PUBLIC_KEY_PATH="${FILE_DIR}/public.key"
+
+GPG_KEY_ID="Pulp QE"
+
+# Export a public key
+/usr/bin/gpg --armor --export "${GPG_KEY_ID}" > $PUBLIC_KEY_PATH
+
+COMMON_GPG_OPTS="--batch --armor --digest-algo SHA256"
+
+# Create a detached signature
+/usr/bin/gpg ${COMMON_GPG_OPTS} \
+  --detach-sign \
+  --output "${SIGNATURE_PATH}" \
+  --local-user "${GPG_KEY_ID}" \
+  "${FILE_PATH}"
+
+# Create an inline signature
+/usr/bin/gpg ${COMMON_GPG_OPTS} \
+  --clearsign \
+  --output "${INLINE_SIGNATURE_PATH}" \
+  --local-user "${GPG_KEY_ID}" \
+  "${FILE_PATH}"
+
+echo {\"file\": \"${FILE_PATH}\", \
+      \"signature\": \"${SIGNATURE_PATH}\", \
+      \"key\": \"${PUBLIC_KEY_PATH}\"}

pulp_deb/tests/functional/utils.py

@@ -38,6 +38,7 @@
     ArtifactsApi,
     Configuration,
     TasksApi,
+    SigningServicesApi,
 )
 from pulpcore.client.pulp_deb import (
     ApiClient as DebApiClient,
@@ -65,6 +66,7 @@
 core_client = CoreApiClient(configuration)
 artifact_api = ArtifactsApi(core_client)
 task_api = TasksApi(core_client)
+signing_service_api = SigningServicesApi(core_client)
 
 deb_client = DebApiClient(configuration)
 deb_generic_content_api = ContentGenericContentsApi(deb_client)

setup.py

@@ -2,7 +2,7 @@
 
 from setuptools import find_packages, setup
 
-requirements = ["pulpcore>=3.1.0.dev,<3.3", "python-debian>=0.1.36"]
+requirements = ["pulpcore>=3.3.0.dev", "python-debian>=0.1.36"]
 
 setup(
     name="pulp-deb",