Add support for automatic publishing and distributing

closes: #7622
https://pulp.plan.io/issues/7622

CHANGES/7622.feature

@@ -0,0 +1 @@
+Add support for automatic publishing and distributing.

pulp_rpm/app/migrations/0032_new_distribution_model.py

@@ -0,0 +1,76 @@
+# Generated by Django 2.2.19 on 2021-03-19 17:42
+
+from django.db import connection, migrations, models, transaction
+import django.db.models.deletion
+
+
+pks_to_delete = []
+
+
+def migrate_data_from_old_master_model_to_new_master_model(apps, schema_editor):
+    RpmDistribution = apps.get_model('rpm', 'RpmDistribution')
+    CoreDistribution = apps.get_model('core', 'Distribution')
+    for old_file_distribution in RpmDistribution.objects.all():
+        with transaction.atomic():
+            new_master_model_entry = CoreDistribution(
+                pulp_id=old_file_distribution.pulp_id,
+                pulp_created=old_file_distribution.pulp_created,
+                pulp_last_updated=old_file_distribution.pulp_last_updated,
+                pulp_type=old_file_distribution.pulp_type,
+                name=old_file_distribution.name,
+                base_path=old_file_distribution.base_path,
+                content_guard=old_file_distribution.content_guard,
+                remote=old_file_distribution.remote,
+                publication=old_file_distribution.publication,
+            )
+            new_master_model_entry.save()
+            old_file_distribution.distribution_ptr = new_master_model_entry
+            old_file_distribution.save()
+            pks_to_delete.append(old_file_distribution.pulp_id)
+
+
+def delete_remaining_old_master_model_entries(apps, schema_editor):
+    with connection.cursor() as cursor:
+        for pk in pks_to_delete:
+            cursor.execute("DELETE from core_basedistribution WHERE pulp_id = %s", [pk])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0062_add_new_distribution_mastermodel'),
+        ('rpm', '0031_ulnremote'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='rpmdistribution',
+            name='distribution_ptr',
+            field=models.OneToOneField(auto_created=True, null=True, default=None,
+                                       on_delete=django.db.models.deletion.CASCADE,
+                                       parent_link=True, primary_key=False,
+                                       related_name='rpm_rpmdistribution', serialize=False,
+                                       to='core.Distribution'),
+            preserve_default=False,
+        ),
+        migrations.RunPython(migrate_data_from_old_master_model_to_new_master_model),
+        migrations.RemoveField(
+            model_name='rpmdistribution',
+            name='basedistribution_ptr',
+        ),
+        migrations.AlterField(
+            model_name='rpmdistribution',
+            name='distribution_ptr',
+            field=models.OneToOneField(auto_created=True,
+                                       on_delete=django.db.models.deletion.CASCADE,
+                                       parent_link=True, primary_key=True,
+                                       related_name='rpm_rpmdistribution', serialize=False,
+                                       to='core.Distribution'),
+            preserve_default=False,
+        ),
+        migrations.RemoveField(
+            model_name='rpmdistribution',
+            name='publication',
+        ),
+        migrations.RunPython(delete_remaining_old_master_model_entries),
+    ]

pulp_rpm/app/migrations/0033_auto_publish.py

@@ -0,0 +1,43 @@
+# Generated by Django 2.2.19 on 2021-04-02 21:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rpm', '0032_new_distribution_model'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='rpmrepository',
+            name='autopublish',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='rpmrepository',
+            name='gpgcheck',
+            field=models.IntegerField(choices=[(0, 0), (1, 1)], default=0),
+        ),
+        migrations.AddField(
+            model_name='rpmrepository',
+            name='metadata_checksum_type',
+            field=models.CharField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], default='sha256', max_length=10),
+        ),
+        migrations.AddField(
+            model_name='rpmrepository',
+            name='package_checksum_type',
+            field=models.CharField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], default='sha256', max_length=10),
+        ),
+        migrations.AddField(
+            model_name='rpmrepository',
+            name='repo_gpgcheck',
+            field=models.IntegerField(choices=[(0, 0), (1, 1)], default=0),
+        ),
+        migrations.AddField(
+            model_name='rpmrepository',
+            name='sqlite_metadata',
+            field=models.BooleanField(default=False),
+        ),
+    ]

pulp_rpm/app/models/repository.py

@@ -18,12 +18,12 @@
     Repository,
     RepositoryVersion,
     Publication,
-    PublicationDistribution,
+    Distribution,
     Task,
 )
 from pulpcore.plugin.repo_version_utils import remove_duplicates, validate_repo_version
 
-from pulp_rpm.app.constants import CHECKSUM_CHOICES
+from pulp_rpm.app.constants import CHECKSUM_CHOICES, CHECKSUM_TYPES
 from pulp_rpm.app.models import (
     DistributionTree,
     Package,
@@ -205,6 +205,7 @@ class RpmRepository(Repository):
         ModulemdDefaults,
     ]
     REMOTE_TYPES = [RpmRemote]
+    GPGCHECK_CHOICES = [(0, 0), (1, 1)]
 
     metadata_signing_service = models.ForeignKey(
         AsciiArmoredDetachedSigningService, on_delete=models.SET_NULL, null=True
@@ -217,6 +218,17 @@ class RpmRepository(Repository):
     original_checksum_types = JSONField(default=dict)
     retain_package_versions = models.PositiveIntegerField(default=0)
 
+    autopublish = models.BooleanField(default=False)
+    metadata_checksum_type = models.CharField(
+        default=CHECKSUM_TYPES.SHA256, choices=CHECKSUM_CHOICES, max_length=10
+    )
+    package_checksum_type = models.CharField(
+        default=CHECKSUM_TYPES.SHA256, choices=CHECKSUM_CHOICES, max_length=10
+    )
+    gpgcheck = models.IntegerField(default=0, choices=GPGCHECK_CHOICES)
+    repo_gpgcheck = models.IntegerField(default=0, choices=GPGCHECK_CHOICES)
+    sqlite_metadata = models.BooleanField(default=False)
+
     def new_version(self, base_version=None):
         """
         Create a new RepositoryVersion for this Repository.
@@ -253,6 +265,36 @@ def new_version(self, base_version=None):
                 resource.save()
             return version
 
+    def on_new_version(self, version):
+        """
+        Called when new repository versions are created.
+
+        Args:
+            version: The new repository version.
+        """
+        super().on_new_version(version)
+
+        # avoid circular import issues
+        from pulp_rpm.app import tasks
+
+        if self.autopublish:
+            publication = tasks.publish(
+                repository_version_pk=version.pk,
+                gpgcheck_options={"gpgcheck": self.gpgcheck, "repo_gpgcheck": self.repo_gpgcheck},
+                metadata_signing_service=self.metadata_signing_service,
+                checksum_types={
+                    "metadata": self.metadata_checksum_type,
+                    "package": self.package_checksum_type,
+                },
+                sqlite_metadata=self.sqlite_metadata,
+            )
+            distributions = self.distributions.all()
+
+            if publication and distributions:
+                for distribution in distributions:
+                    distribution.publication = publication
+                    distribution.save()
+
     @staticmethod
     def artifacts_for_version(version):
         """
@@ -394,7 +436,7 @@ class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
 
 
-class RpmDistribution(PublicationDistribution):
+class RpmDistribution(Distribution):
     """
     Distribution for "rpm" content.
     """

pulp_rpm/app/serializers/repository.py

@@ -7,10 +7,12 @@
 from pulpcore.plugin.models import (
     AsciiArmoredDetachedSigningService,
     Remote,
+    Publication,
 )
 from pulpcore.plugin.serializers import (
     RelatedField,
-    PublicationDistributionSerializer,
+    DetailRelatedField,
+    DistributionSerializer,
     PublicationSerializer,
     RemoteSerializer,
     RepositorySerializer,
@@ -39,6 +41,14 @@ class RpmRepositorySerializer(RepositorySerializer):
     Serializer for Rpm Repositories.
     """
 
+    autopublish = serializers.BooleanField(
+        help_text=_(
+            "Whether to automatically create publications for new repository versions, "
+            "and update any distributions pointing to this repository."
+        ),
+        default=False,
+        required=False,
+    )
     metadata_signing_service = RelatedField(
         help_text="A reference to an associated signing service.",
         view_name="signing-services-detail",
@@ -56,11 +66,62 @@ class RpmRepositorySerializer(RepositorySerializer):
         min_value=0,
         required=False,
     )
+    metadata_checksum_type = serializers.ChoiceField(
+        help_text=_("The checksum type for metadata."),
+        choices=CHECKSUM_CHOICES,
+        default=CHECKSUM_TYPES.SHA256,
+    )
+    package_checksum_type = serializers.ChoiceField(
+        help_text=_("The checksum type for packages."),
+        choices=CHECKSUM_CHOICES,
+        default=CHECKSUM_TYPES.SHA256,
+    )
+    gpgcheck = serializers.IntegerField(
+        max_value=1,
+        min_value=0,
+        default=0,
+        required=False,
+        help_text=_(
+            "An option specifying whether a client should perform "
+            "a GPG signature check on packages."
+        ),
+    )
+    repo_gpgcheck = serializers.IntegerField(
+        max_value=1,
+        min_value=0,
+        default=0,
+        required=False,
+        help_text=_(
+            "An option specifying whether a client should perform "
+            "a GPG signature check on the repodata."
+        ),
+    )
+    sqlite_metadata = serializers.BooleanField(
+        default=False,
+        required=False,
+        help_text=_("An option specifying whether Pulp should generate SQLite metadata."),
+    )
+
+    def validate(self, data):
+        """Validate data."""
+        if (
+            data["metadata_checksum_type"] not in settings.ALLOWED_CONTENT_CHECKSUMS
+            or data["package_checksum_type"] not in settings.ALLOWED_CONTENT_CHECKSUMS
+        ):
+            raise serializers.ValidationError(_(ALLOWED_CHECKSUM_ERROR_MSG))
+        validated_data = super().validate(data)
+        return validated_data
 
     class Meta:
         fields = RepositorySerializer.Meta.fields + (
+            "autopublish",
             "metadata_signing_service",
             "retain_package_versions",
+            "metadata_checksum_type",
+            "package_checksum_type",
+            "gpgcheck",
+            "repo_gpgcheck",
+            "sqlite_metadata",
         )
         model = RpmRepository
 
@@ -195,13 +256,21 @@ class Meta:
         model = RpmPublication
 
 
-class RpmDistributionSerializer(PublicationDistributionSerializer):
+class RpmDistributionSerializer(DistributionSerializer):
     """
     Serializer for RPM Distributions.
     """
 
+    publication = DetailRelatedField(
+        required=False,
+        help_text=_("Publication to be served"),
+        view_name_pattern=r"publications(-.*/.*)?-detail",
+        queryset=Publication.objects.exclude(complete=False),
+        allow_null=True,
+    )
+
     class Meta:
-        fields = PublicationDistributionSerializer.Meta.fields
+        fields = DistributionSerializer.Meta.fields + ("publication",)
         model = RpmDistribution
 
 

pulp_rpm/app/tasks/publishing.py

@@ -332,6 +332,10 @@ def publish(
                     metadata_signing_service=metadata_signing_service,
                 )
 
+        log.info(_("Publication: {publication} created").format(publication=publication.pk))
+
+        return publication
+
 
 def create_repomd_xml(
     content,