Add support for automatic publishing and distributing

Required PR: pulp/pulpcore#1084

closes: #7622
https://pulp.plan.io/issues/7622

CHANGES/7622.feature

@@ -0,0 +1 @@
+Add support for automatic publishing and distributing.

pulp_rpm/app/migrations/0031_auto_publish.py

@@ -0,0 +1,51 @@
+# Generated by Django 2.2.18 on 2021-02-15 22:51
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+def populate_publish_settings(apps, schema_editor):
+    RpmPublication = apps.get_model('rpm', 'rpmpublication')
+    RpmPublishSettings = apps.get_model('rpm', 'rpmpublishsettings')
+
+    for publication in RpmPublication.objects.all():
+        publish_settings, _ = RpmPublishSettings.objects.get_or_create(
+            metadata_checksum_type=publication.metadata_checksum_type,
+            package_checksum_type=publication.package_checksum_type,
+            gpgcheck=publication.gpgcheck,
+            repo_gpgcheck=publication.repo_gpgcheck,
+            sqlite_metadata=publication.sqlite_metadata,
+        )
+        publication.publish_settings = publish_settings
+        publication.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0059_publish_settings'),
+        ('rpm', '0030_DATA_fix_updaterecord'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RpmPublishSettings',
+            fields=[
+                ('publishsettings_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='rpm_rpmpublishsettings', serialize=False, to='core.PublishSettings')),
+                ('metadata_checksum_type', models.CharField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], max_length=10)),
+                ('package_checksum_type', models.CharField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], max_length=10)),
+                ('gpgcheck', models.IntegerField(choices=[(0, 0), (1, 1)], default=0)),
+                ('repo_gpgcheck', models.IntegerField(choices=[(0, 0), (1, 1)], default=0)),
+                ('sqlite_metadata', models.BooleanField(default=False)),
+            ],
+            options={
+                'default_related_name': '%(app_label)s_%(model_name)s',
+            },
+            bases=('core.publishsettings',),
+        ),
+        migrations.AlterUniqueTogether(
+            name='rpmpublishsettings',
+            unique_together={('metadata_checksum_type', 'package_checksum_type', 'gpgcheck', 'repo_gpgcheck', 'sqlite_metadata')},
+        ),
+        migrations.RunPython(populate_publish_settings),
+    ]

pulp_rpm/app/migrations/0032_migrate_abstract_fields.py

@@ -0,0 +1,30 @@
+# Generated by Django 2.2.19 on 2021-02-23 18:05
+
+from django.db import migrations
+
+
+def migrate_distribution_fields(apps, schema_editor):
+    # Set the core "_publication" field with the value of the detail "publication" field
+    RpmDistribution = apps.get_model('rpm', 'rpmdistribution')
+
+    distributions = list(RpmDistribution.objects.all())
+
+    for distribution in distributions:
+        distribution._publication = distribution.publication
+
+    RpmDistribution.objects.bulk_update(distributions, ['_publication'])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rpm', '0031_auto_publish'),
+        ('core', '0060_add_new_distribution_fields'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='rpmdistribution',
+            name='publication',
+        ),
+    ]

pulp_rpm/app/models/__init__.py

@@ -9,4 +9,10 @@
 from .distribution import Addon, Checksum, DistributionTree, Image, Variant  # noqa
 from .modulemd import Modulemd, ModulemdDefaults  # noqa
 from .package import Package  # noqa
-from .repository import RpmDistribution, RpmPublication, RpmRemote, RpmRepository  # noqa
+from .repository import (  # noqa
+    RpmDistribution,
+    RpmPublication,
+    RpmPublishSettings,
+    RpmRemote,
+    RpmRepository,
+)

pulp_rpm/app/models/repository.py

@@ -18,6 +18,7 @@
     Repository,
     RepositoryVersion,
     Publication,
+    PublishSettings,
     PublicationDistribution,
     Task,
 )
@@ -102,6 +103,32 @@ class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
 
 
+class RpmPublishSettings(PublishSettings):
+    """
+    Publication for "rpm" content.
+    """
+
+    TYPE = "rpm"
+
+    GPGCHECK_CHOICES = [(0, 0), (1, 1)]
+
+    metadata_checksum_type = models.CharField(choices=CHECKSUM_CHOICES, max_length=10)
+    package_checksum_type = models.CharField(choices=CHECKSUM_CHOICES, max_length=10)
+    gpgcheck = models.IntegerField(default=0, choices=GPGCHECK_CHOICES)
+    repo_gpgcheck = models.IntegerField(default=0, choices=GPGCHECK_CHOICES)
+    sqlite_metadata = models.BooleanField(default=False)
+
+    class Meta:
+        default_related_name = "%(app_label)s_%(model_name)s"
+        unique_together = [
+            "metadata_checksum_type",
+            "package_checksum_type",
+            "gpgcheck",
+            "repo_gpgcheck",
+            "sqlite_metadata",
+        ]
+
+
 class RpmRepository(Repository):
     """
     Repository for "rpm" content.
@@ -184,6 +211,30 @@ def new_version(self, base_version=None):
                 resource.save()
             return version
 
+    def on_new_version(self, version):
+        """
+        Called when new repository versions are created.
+
+        Args:
+            version: The new repository version.
+        """
+        super().on_new_version(version)
+
+        # avoid circular import issues
+        from pulp_rpm.app import tasks
+
+        if self.publish_settings:
+            publication = tasks.publish(
+                repository_version_pk=version.pk,
+                publish_settings_pk=self.publish_settings.pk,
+            )
+            distributions = self.distributions.all()
+
+            if publication and distributions:
+                for distribution in distributions:
+                    distribution.publication = publication
+                    distribution.save()
+
     @staticmethod
     def artifacts_for_version(version):
         """

pulp_rpm/app/serializers/__init__.py

@@ -23,6 +23,7 @@
     CopySerializer,
     RpmDistributionSerializer,
     RpmPublicationSerializer,
+    RpmPublishSettingsSerializer,
     RpmRemoteSerializer,
     RpmRepositorySerializer,
     RpmRepositorySyncURLSerializer,

pulp_rpm/app/serializers/repository.py

@@ -12,6 +12,7 @@
     RelatedField,
     PublicationDistributionSerializer,
     PublicationSerializer,
+    PublishSettingsSerializer,
     RemoteSerializer,
     RepositorySerializer,
     RepositorySyncURLSerializer,
@@ -29,6 +30,7 @@
     RpmRemote,
     RpmRepository,
     RpmPublication,
+    RpmPublishSettings,
 )
 from pulp_rpm.app.schema import COPY_CONFIG_SCHEMA
 
@@ -89,6 +91,68 @@ class Meta:
         model = RpmRemote
 
 
+class RpmPublishSettingsSerializer(PublishSettingsSerializer):
+    """
+    A Serializer for RpmPublication.
+    """
+
+    metadata_checksum_type = serializers.ChoiceField(
+        help_text=_("The checksum type for metadata."),
+        choices=CHECKSUM_CHOICES,
+        default=CHECKSUM_TYPES.SHA256,
+    )
+    package_checksum_type = serializers.ChoiceField(
+        help_text=_("The checksum type for packages."),
+        choices=CHECKSUM_CHOICES,
+        default=CHECKSUM_TYPES.SHA256,
+    )
+    gpgcheck = serializers.IntegerField(
+        max_value=1,
+        min_value=0,
+        default=0,
+        required=False,
+        help_text=_(
+            "An option specifying whether a client should perform "
+            "a GPG signature check on packages."
+        ),
+    )
+    repo_gpgcheck = serializers.IntegerField(
+        max_value=1,
+        min_value=0,
+        default=0,
+        required=False,
+        help_text=_(
+            "An option specifying whether a client should perform "
+            "a GPG signature check on the repodata."
+        ),
+    )
+    sqlite_metadata = serializers.BooleanField(
+        default=False,
+        required=False,
+        help_text=_("An option specifying whether Pulp should generate SQLite metadata."),
+    )
+
+    def validate(self, data):
+        """Validate data."""
+        if (
+            data["metadata_checksum_type"] not in settings.ALLOWED_CONTENT_CHECKSUMS
+            or data["package_checksum_type"] not in settings.ALLOWED_CONTENT_CHECKSUMS
+        ):
+            raise serializers.ValidationError(_(ALLOWED_CHECKSUM_ERROR_MSG))
+        validated_data = super().validate(data)
+        return validated_data
+
+    class Meta:
+        fields = PublishSettingsSerializer.Meta.fields + (
+            "metadata_checksum_type",
+            "package_checksum_type",
+            "gpgcheck",
+            "repo_gpgcheck",
+            "sqlite_metadata",
+        )
+        model = RpmPublishSettings
+
+
 class RpmPublicationSerializer(PublicationSerializer):
     """
     A Serializer for RpmPublication.

pulp_rpm/app/tasks/publishing.py

@@ -35,7 +35,9 @@
     PackageGroup,
     PackageLangpacks,
     RepoMetadataFile,
+    RpmRepository,
     RpmPublication,
+    RpmPublishSettings,
     UpdateRecord,
 )
 
@@ -259,6 +261,7 @@ def get_checksum_type(name, checksum_types):
 
 def publish(
     repository_version_pk,
+    publish_settings_pk=None,
     gpgcheck_options=None,
     metadata_signing_service=None,
     checksum_types=None,
@@ -269,6 +272,7 @@ def publish(
 
     Args:
         repository_version_pk (str): Create a publication from this repository version.
+        publish_settings_pk (str): Create a publication from these publish settings.
         gpgcheck_options (dict): GPG signature check options.
         metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService):
             A reference to an associated signing service.
@@ -277,8 +281,26 @@ def publish(
 
     """
     repository_version = RepositoryVersion.objects.get(pk=repository_version_pk)
-    repository = repository_version.repository.cast()
-    checksum_types = checksum_types or {}
+    # TODO: we can't make publish_settings_pk mandatory due to the migration plugin
+    # calling publish() directly, therefore we need a lot of compatibility shims.
+    # Try to remove these in the future once migration is no longer a concern.
+
+    if publish_settings_pk:
+        publish_settings = RpmPublishSettings.objects.get(pk=publish_settings_pk)
+    else:
+        publish_settings = RpmPublishSettings.objects.get_or_create(
+            gpgcheck=gpgcheck_options.get("gpgcheck") if gpgcheck_options else None,
+            repo_gpgcheck=gpgcheck_options.get("repo_gpgcheck") if gpgcheck_options else None,
+            metadata_checksum_type=checksum_types.get("metadata") if checksum_types else None,
+            package_checksum_type=checksum_types.get("package") if checksum_types else None,
+            sqlite_metadata=sqlite_metadata,
+        )
+
+    repository = RpmRepository.objects.get(pk=repository_version.repository.pk)
+    checksum_types = dict(
+        metadata=publish_settings.metadata_checksum_type,
+        package=publish_settings.package_checksum_type,
+    )
 
     checksum_types["original"] = repository.original_checksum_types
     original_metadata_checksum_type = repository.original_checksum_types.get(
@@ -293,18 +315,15 @@ def publish(
     )
 
     with WorkingDirectory():
-        with RpmPublication.create(repository_version) as publication:
+        with RpmPublication.create(repository_version, publish_settings) as publication:
             publication.package_checksum_type = checksum_types.get("package", CHECKSUM_TYPES.SHA256)
             publication.metadata_checksum_type = checksum_types.get(
                 "metadata", original_metadata_checksum_type
             )
 
-            if gpgcheck_options is not None:
-                publication.gpgcheck = gpgcheck_options.get("gpgcheck")
-                publication.repo_gpgcheck = gpgcheck_options.get("repo_gpgcheck")
-
-            if sqlite_metadata:
-                publication.sqlite_metadata = True
+            publication.gpgcheck = publish_settings.gpgcheck
+            publication.repo_gpgcheck = publish_settings.repo_gpgcheck
+            publication.sqlite_metadata = publish_settings.sqlite_metadata
 
             publication_data = PublicationData(publication)
             publication_data.populate()
@@ -317,7 +336,7 @@ def publish(
                 publication,
                 checksum_types,
                 publication_data.repomdrecords,
-                metadata_signing_service=metadata_signing_service,
+                metadata_signing_service=repository.metadata_signing_service,
             )
 
             for sub_repo in publication_data.sub_repos:
@@ -331,9 +350,13 @@ def publish(
                     checksum_types,
                     extra_repomdrecords,
                     name,
-                    metadata_signing_service=metadata_signing_service,
+                    metadata_signing_service=repository.metadata_signing_service,
                 )
 
+        log.info(_("Publication: {publication} created").format(publication=publication.pk))
+
+        return publication
+
 
 def create_repomd_xml(
     content,