diff --git a/CHANGES/7622.feature b/CHANGES/7622.feature new file mode 100644 index 0000000000..d8490631c6 --- /dev/null +++ b/CHANGES/7622.feature @@ -0,0 +1 @@ +Add support for automatic publishing and distributing. diff --git a/pulp_rpm/app/migrations/0032_new_distribution_model.py b/pulp_rpm/app/migrations/0032_new_distribution_model.py new file mode 100644 index 0000000000..9e82d306c1 --- /dev/null +++ b/pulp_rpm/app/migrations/0032_new_distribution_model.py @@ -0,0 +1,76 @@ +# Generated by Django 2.2.19 on 2021-03-19 17:42 + +from django.db import connection, migrations, models, transaction +import django.db.models.deletion + + +pks_to_delete = [] + + +def migrate_data_from_old_master_model_to_new_master_model(apps, schema_editor): + RpmDistribution = apps.get_model('rpm', 'RpmDistribution') + CoreDistribution = apps.get_model('core', 'Distribution') + for old_file_distribution in RpmDistribution.objects.all(): + with transaction.atomic(): + new_master_model_entry = CoreDistribution( + pulp_id=old_file_distribution.pulp_id, + pulp_created=old_file_distribution.pulp_created, + pulp_last_updated=old_file_distribution.pulp_last_updated, + pulp_type=old_file_distribution.pulp_type, + name=old_file_distribution.name, + base_path=old_file_distribution.base_path, + content_guard=old_file_distribution.content_guard, + remote=old_file_distribution.remote, + publication=old_file_distribution.publication, + ) + new_master_model_entry.save() + old_file_distribution.distribution_ptr = new_master_model_entry + old_file_distribution.save() + pks_to_delete.append(old_file_distribution.pulp_id) + + +def delete_remaining_old_master_model_entries(apps, schema_editor): + with connection.cursor() as cursor: + for pk in pks_to_delete: + cursor.execute("DELETE from core_basedistribution WHERE pulp_id = %s", [pk]) + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0062_add_new_distribution_mastermodel'), + ('rpm', '0031_ulnremote'), + ] + + operations = [ + migrations.AddField( + model_name='rpmdistribution', + name='distribution_ptr', + field=models.OneToOneField(auto_created=True, null=True, default=None, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, primary_key=False, + related_name='rpm_rpmdistribution', serialize=False, + to='core.Distribution'), + preserve_default=False, + ), + migrations.RunPython(migrate_data_from_old_master_model_to_new_master_model), + migrations.RemoveField( + model_name='rpmdistribution', + name='basedistribution_ptr', + ), + migrations.AlterField( + model_name='rpmdistribution', + name='distribution_ptr', + field=models.OneToOneField(auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, primary_key=True, + related_name='rpm_rpmdistribution', serialize=False, + to='core.Distribution'), + preserve_default=False, + ), + migrations.RemoveField( + model_name='rpmdistribution', + name='publication', + ), + migrations.RunPython(delete_remaining_old_master_model_entries), + ] diff --git a/pulp_rpm/app/migrations/0033_auto_publish.py b/pulp_rpm/app/migrations/0033_auto_publish.py new file mode 100644 index 0000000000..1f4717979d --- /dev/null +++ b/pulp_rpm/app/migrations/0033_auto_publish.py @@ -0,0 +1,43 @@ +# Generated by Django 2.2.19 on 2021-04-02 21:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('rpm', '0032_new_distribution_model'), + ] + + operations = [ + migrations.AddField( + model_name='rpmrepository', + name='autopublish', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='rpmrepository', + name='gpgcheck', + field=models.IntegerField(choices=[(0, 0), (1, 1)], default=0), + ), + migrations.AddField( + model_name='rpmrepository', + name='metadata_checksum_type', + field=models.CharField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], default='sha256', max_length=10), + ), + migrations.AddField( + model_name='rpmrepository', + name='package_checksum_type', + field=models.CharField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], default='sha256', max_length=10), + ), + migrations.AddField( + model_name='rpmrepository', + name='repo_gpgcheck', + field=models.IntegerField(choices=[(0, 0), (1, 1)], default=0), + ), + migrations.AddField( + model_name='rpmrepository', + name='sqlite_metadata', + field=models.BooleanField(default=False), + ), + ] diff --git a/pulp_rpm/app/models/repository.py b/pulp_rpm/app/models/repository.py index ac4dacfa5d..29da241195 100644 --- a/pulp_rpm/app/models/repository.py +++ b/pulp_rpm/app/models/repository.py @@ -18,12 +18,12 @@ Repository, RepositoryVersion, Publication, - PublicationDistribution, + Distribution, Task, ) from pulpcore.plugin.repo_version_utils import remove_duplicates, validate_repo_version -from pulp_rpm.app.constants import CHECKSUM_CHOICES +from pulp_rpm.app.constants import CHECKSUM_CHOICES, CHECKSUM_TYPES from pulp_rpm.app.models import ( DistributionTree, Package, @@ -205,6 +205,7 @@ class RpmRepository(Repository): ModulemdDefaults, ] REMOTE_TYPES = [RpmRemote] + GPGCHECK_CHOICES = [(0, 0), (1, 1)] metadata_signing_service = models.ForeignKey( AsciiArmoredDetachedSigningService, on_delete=models.SET_NULL, null=True @@ -217,6 +218,17 @@ class RpmRepository(Repository): original_checksum_types = JSONField(default=dict) retain_package_versions = models.PositiveIntegerField(default=0) + autopublish = models.BooleanField(default=False) + metadata_checksum_type = models.CharField( + default=CHECKSUM_TYPES.SHA256, choices=CHECKSUM_CHOICES, max_length=10 + ) + package_checksum_type = models.CharField( + default=CHECKSUM_TYPES.SHA256, choices=CHECKSUM_CHOICES, max_length=10 + ) + gpgcheck = models.IntegerField(default=0, choices=GPGCHECK_CHOICES) + repo_gpgcheck = models.IntegerField(default=0, choices=GPGCHECK_CHOICES) + sqlite_metadata = models.BooleanField(default=False) + def new_version(self, base_version=None): """ Create a new RepositoryVersion for this Repository. @@ -253,6 +265,36 @@ def new_version(self, base_version=None): resource.save() return version + def on_new_version(self, version): + """ + Called when new repository versions are created. + + Args: + version: The new repository version. + """ + super().on_new_version(version) + + # avoid circular import issues + from pulp_rpm.app import tasks + + if self.autopublish: + publication = tasks.publish( + repository_version_pk=version.pk, + gpgcheck_options={"gpgcheck": self.gpgcheck, "repo_gpgcheck": self.repo_gpgcheck}, + metadata_signing_service=self.metadata_signing_service, + checksum_types={ + "metadata": self.metadata_checksum_type, + "package": self.package_checksum_type, + }, + sqlite_metadata=self.sqlite_metadata, + ) + distributions = self.distributions.all() + + if publication and distributions: + for distribution in distributions: + distribution.publication = publication + distribution.save() + @staticmethod def artifacts_for_version(version): """ @@ -394,7 +436,7 @@ class Meta: default_related_name = "%(app_label)s_%(model_name)s" -class RpmDistribution(PublicationDistribution): +class RpmDistribution(Distribution): """ Distribution for "rpm" content. """ diff --git a/pulp_rpm/app/serializers/repository.py b/pulp_rpm/app/serializers/repository.py index 3abc670d05..0da4a03413 100644 --- a/pulp_rpm/app/serializers/repository.py +++ b/pulp_rpm/app/serializers/repository.py @@ -7,10 +7,12 @@ from pulpcore.plugin.models import ( AsciiArmoredDetachedSigningService, Remote, + Publication, ) from pulpcore.plugin.serializers import ( RelatedField, - PublicationDistributionSerializer, + DetailRelatedField, + DistributionSerializer, PublicationSerializer, RemoteSerializer, RepositorySerializer, @@ -39,6 +41,14 @@ class RpmRepositorySerializer(RepositorySerializer): Serializer for Rpm Repositories. """ + autopublish = serializers.BooleanField( + help_text=_( + "Whether to automatically create publications for new repository versions, " + "and update any distributions pointing to this repository." + ), + default=False, + required=False, + ) metadata_signing_service = RelatedField( help_text="A reference to an associated signing service.", view_name="signing-services-detail", @@ -56,11 +66,62 @@ class RpmRepositorySerializer(RepositorySerializer): min_value=0, required=False, ) + metadata_checksum_type = serializers.ChoiceField( + help_text=_("The checksum type for metadata."), + choices=CHECKSUM_CHOICES, + default=CHECKSUM_TYPES.SHA256, + ) + package_checksum_type = serializers.ChoiceField( + help_text=_("The checksum type for packages."), + choices=CHECKSUM_CHOICES, + default=CHECKSUM_TYPES.SHA256, + ) + gpgcheck = serializers.IntegerField( + max_value=1, + min_value=0, + default=0, + required=False, + help_text=_( + "An option specifying whether a client should perform " + "a GPG signature check on packages." + ), + ) + repo_gpgcheck = serializers.IntegerField( + max_value=1, + min_value=0, + default=0, + required=False, + help_text=_( + "An option specifying whether a client should perform " + "a GPG signature check on the repodata." + ), + ) + sqlite_metadata = serializers.BooleanField( + default=False, + required=False, + help_text=_("An option specifying whether Pulp should generate SQLite metadata."), + ) + + def validate(self, data): + """Validate data.""" + if ( + data["metadata_checksum_type"] not in settings.ALLOWED_CONTENT_CHECKSUMS + or data["package_checksum_type"] not in settings.ALLOWED_CONTENT_CHECKSUMS + ): + raise serializers.ValidationError(_(ALLOWED_CHECKSUM_ERROR_MSG)) + validated_data = super().validate(data) + return validated_data class Meta: fields = RepositorySerializer.Meta.fields + ( + "autopublish", "metadata_signing_service", "retain_package_versions", + "metadata_checksum_type", + "package_checksum_type", + "gpgcheck", + "repo_gpgcheck", + "sqlite_metadata", ) model = RpmRepository @@ -195,13 +256,21 @@ class Meta: model = RpmPublication -class RpmDistributionSerializer(PublicationDistributionSerializer): +class RpmDistributionSerializer(DistributionSerializer): """ Serializer for RPM Distributions. """ + publication = DetailRelatedField( + required=False, + help_text=_("Publication to be served"), + view_name_pattern=r"publications(-.*/.*)?-detail", + queryset=Publication.objects.exclude(complete=False), + allow_null=True, + ) + class Meta: - fields = PublicationDistributionSerializer.Meta.fields + fields = DistributionSerializer.Meta.fields + ("publication",) model = RpmDistribution diff --git a/pulp_rpm/app/tasks/publishing.py b/pulp_rpm/app/tasks/publishing.py index 4e24420996..98e78f11c7 100644 --- a/pulp_rpm/app/tasks/publishing.py +++ b/pulp_rpm/app/tasks/publishing.py @@ -332,6 +332,10 @@ def publish( metadata_signing_service=metadata_signing_service, ) + log.info(_("Publication: {publication} created").format(publication=publication.pk)) + + return publication + def create_repomd_xml( content, diff --git a/pulp_rpm/app/tasks/synchronizing.py b/pulp_rpm/app/tasks/synchronizing.py index 196b571303..36fbd766b6 100644 --- a/pulp_rpm/app/tasks/synchronizing.py +++ b/pulp_rpm/app/tasks/synchronizing.py @@ -180,7 +180,7 @@ def is_optimized_sync(repository, remote, url): and repository.last_sync_repomd_checksum == repomd_checksum ) if is_optimized: - optimize_data = dict(message="Optimizing Sync", code="optimizing.sync") + optimize_data = dict(message="Optimizing Sync", code="sync.optimizing") with ProgressReport(**optimize_data) as optimize_pb: optimize_pb.done = 1 optimize_pb.save() @@ -271,10 +271,12 @@ def synchronize(remote_pk, repository_pk, mirror, skip_types, optimize): new_url=remote_url, ) dv = RpmDeclarativeVersion(first_stage=first_stage, repository=repository, mirror=mirror) - dv.create() - repository.last_sync_remote = remote - repository.last_sync_repo_version = repository.latest_version().number - repository.save() + version = dv.create() + if version: + repository.last_sync_remote = remote + repository.last_sync_repo_version = version.number + repository.save() + return version class RpmDeclarativeVersion(DeclarativeVersion): @@ -423,7 +425,7 @@ def newpkgcb(pkgId, name, arch): async def run(self): """Build `DeclarativeContent` from the repodata.""" - progress_data = dict(message="Downloading Metadata Files", code="downloading.metadata") + progress_data = dict(message="Downloading Metadata Files", code="sync.downloading.metadata") with ProgressReport(**progress_data) as metadata_pb: self.data.metadata_pb = metadata_pb @@ -577,7 +579,7 @@ async def parse_advisories(self, result): async def _parse_packages(self, packages): progress_data = { "message": "Parsed Packages", - "code": "parsing.packages", + "code": "sync.parsing.packages", "total": len(packages), } @@ -617,7 +619,7 @@ async def _parse_packages(self, packages): async def _parse_advisories(self, updates): progress_data = { "message": "Parsed Advisories", - "code": "parsing.advisories", + "code": "sync.parsing.advisories", "total": len(updates), } with ProgressReport(**progress_data) as advisories_pb: @@ -762,7 +764,7 @@ def _parse_modulemd_list(self, modulemd_index): # Parsing modules happens all at one time, and from here on no useful work happens. # So just report that it finished this stage. - modulemd_pb_data = {"message": "Parsed Modulemd", "code": "parsing.modulemds"} + modulemd_pb_data = {"message": "Parsed Modulemd", "code": "sync.parsing.modulemds"} with ProgressReport(**modulemd_pb_data) as modulemd_pb: modulemd_total = len(modulemd_all) modulemd_pb.total = modulemd_total @@ -799,7 +801,7 @@ def _parse_modulemd_default_names(self, modulemd_index): # work happens. So just report that it finished this stage. modulemd_defaults_pb_data = { "message": "Parsed Modulemd-defaults", - "code": "parsing.modulemd_defaults", + "code": "sync.parsing.modulemd_defaults", } with ProgressReport(**modulemd_defaults_pb_data) as modulemd_defaults_pb: modulemd_defaults_total = len(modulemd_default_names) @@ -845,7 +847,7 @@ def parse(self): comps = libcomps.Comps() comps.fromxml_f(self.comps_result.path) - with ProgressReport(message="Parsed Comps", code="parsing.comps") as comps_pb: + with ProgressReport(message="Parsed Comps", code="sync.parsing.comps") as comps_pb: comps_total = len(comps.groups) + len(comps.categories) + len(comps.environments) comps_pb.total = comps_total comps_pb.done = comps_total diff --git a/pulp_rpm/app/viewsets.py b/pulp_rpm/app/viewsets.py index 5ff5923d34..2d78ee96d8 100644 --- a/pulp_rpm/app/viewsets.py +++ b/pulp_rpm/app/viewsets.py @@ -11,7 +11,7 @@ from pulpcore.plugin.tasking import enqueue_with_reservation from pulpcore.plugin.serializers import AsyncOperationResponseSerializer from pulpcore.plugin.viewsets import ( - BaseDistributionViewSet, + DistributionViewSet, ContentFilter, NoArtifactContentUploadViewSet, NamedModelViewSet, @@ -231,17 +231,21 @@ def create(self, request): serializer.is_valid(raise_exception=True) repository_version = serializer.validated_data.get("repository_version") repository = RpmRepository.objects.get(pk=repository_version.repository.pk) - metadata_checksum_type = serializer.validated_data.get("metadata_checksum_type", "") - package_checksum_type = serializer.validated_data.get("package_checksum_type", "") + metadata_checksum_type = serializer.validated_data.get( + "metadata_checksum_type", self.metadata_checksum_type + ) + package_checksum_type = serializer.validated_data.get( + "package_checksum_type", self.package_checksum_type + ) checksum_types = dict( metadata=metadata_checksum_type, package=package_checksum_type, ) gpgcheck_options = dict( - gpgcheck=serializer.validated_data.get("gpgcheck"), - repo_gpgcheck=serializer.validated_data.get("repo_gpgcheck"), + gpgcheck=serializer.validated_data.get("gpgcheck", self.gpgcheck), + repo_gpgcheck=serializer.validated_data.get("repo_gpgcheck", self.repo_gpgcheck), ) - sqlite_metadata = serializer.validated_data.get("sqlite_metadata", "") + sqlite_metadata = serializer.validated_data.get("sqlite_metadata", self.sqlite_metadata) result = enqueue_with_reservation( tasks.publish, @@ -257,7 +261,7 @@ def create(self, request): return OperationPostponedResponse(result, request) -class RpmDistributionViewSet(BaseDistributionViewSet): +class RpmDistributionViewSet(DistributionViewSet): """ ViewSet for RPM Distributions. """ diff --git a/pulp_rpm/tests/functional/api/test_auto_publish.py b/pulp_rpm/tests/functional/api/test_auto_publish.py new file mode 100644 index 0000000000..5a3b0a2546 --- /dev/null +++ b/pulp_rpm/tests/functional/api/test_auto_publish.py @@ -0,0 +1,104 @@ +# coding=utf-8 +"""Tests that sync file plugin repositories.""" +import unittest + +from pulp_smash import config +from pulp_smash.pulp3.bindings import monitor_task +from pulp_smash.pulp3.utils import gen_repo + +from pulp_rpm.tests.functional.utils import gen_rpm_client, gen_rpm_remote +from pulp_rpm.tests.functional.utils import set_up_module as setUpModule # noqa:F401 + +from pulpcore.client.pulp_rpm import ( + ContentPackagesApi, + DistributionsRpmApi, + PublicationsRpmApi, + RepositoriesRpmApi, + RpmRepositorySyncURL, + RemotesRpmApi, +) + + +class AutoPublishDistributeTestCase(unittest.TestCase): + """Test auto-publish and auto-distribution.""" + + @classmethod + def setUpClass(cls): + """Create class-wide variables.""" + cls.cfg = config.get_config() + cls.client = gen_rpm_client() + + cls.content_api = ContentPackagesApi(cls.client) + cls.repo_api = RepositoriesRpmApi(cls.client) + cls.remote_api = RemotesRpmApi(cls.client) + cls.publications_api = PublicationsRpmApi(cls.client) + cls.distributions_api = DistributionsRpmApi(cls.client) + + def setUp(self): + """Create remote, repo, publish settings, and distribution.""" + self.remote = self.remote_api.create(gen_rpm_remote()) + self.repo = self.repo_api.create(gen_repo(autopublish=True, sqlite_metadata=True)) + response = self.distributions_api.create( + {"name": "foo", "base_path": "bar/foo", "repository": self.repo.pulp_href} + ) + distribution_href = monitor_task(response.task).created_resources[0] + self.distribution = self.distributions_api.read(distribution_href) + + def tearDown(self): + """Clean up.""" + self.repo_api.delete(self.repo.pulp_href) + self.remote_api.delete(self.remote.pulp_href) + self.distributions_api.delete(self.distribution.pulp_href) + + def test_01_sync(self): + """Assert that syncing the repository triggers auto-publish and auto-distribution.""" + self.assertEqual(self.publications_api.list().count, 0) + self.assertTrue(self.distribution.publication is None) + + # Sync the repository. + repository_sync_data = RpmRepositorySyncURL(remote=self.remote.pulp_href) + sync_response = self.repo_api.sync(self.repo.pulp_href, repository_sync_data) + task = monitor_task(sync_response.task) + self.distribution = self.distributions_api.read(self.distribution.pulp_href) + + # Check that all the appropriate resources were created + self.assertGreater(len(task.created_resources), 1) + self.assertEqual(self.publications_api.list().count, 1) + self.assertTrue(self.distribution.publication is not None) + self.assertTrue(self.distribution.publication in task.created_resources) + + # Check that the publish settings were used + publication = self.publications_api.read(self.distribution.publication) + self.assertEqual(publication.sqlite_metadata, True) + + # Sync the repository again. Since there should be no new repository version, there + # should be no new publications or distributions either. + sync_response = self.repo_api.sync(self.repo.pulp_href, repository_sync_data) + task = monitor_task(sync_response.task) + + self.assertEqual(len(task.created_resources), 0) + self.assertEqual(self.publications_api.list().count, 1) + + def test_02_modify(self): + """Assert that modifying the repository triggers auto-publish and auto-distribution.""" + self.assertEqual(self.publications_api.list().count, 0) + self.assertTrue(self.distribution.publication is None) + + # Modify the repository by adding a coment unit + content = self.content_api.list().results[0].pulp_href + + modify_response = self.repo_api.modify( + self.repo.pulp_href, {"add_content_units": [content]} + ) + task = monitor_task(modify_response.task) + self.distribution = self.distributions_api.read(self.distribution.pulp_href) + + # Check that all the appropriate resources were created + self.assertGreater(len(task.created_resources), 1) + self.assertEqual(self.publications_api.list().count, 1) + self.assertTrue(self.distribution.publication is not None) + self.assertTrue(self.distribution.publication in task.created_resources) + + # Check that the publish settings were used + publication = self.publications_api.read(self.distribution.publication) + self.assertEqual(publication.sqlite_metadata, True) diff --git a/requirements.txt b/requirements.txt index c5a580dd41..5eac1a0489 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ django_readonly_field jsonschema>=3.0 libcomps~=0.1.15 productmd>=1.25 -pulpcore>=3.10 +pulpcore>=3.12.0 PyGObject~=3.22 solv~=0.7.17 aiohttp_xmlrpc