Ensure no duplicates are present in a repo version.

Also removes one_shot_upload as it's not used anymore.

closes #4898
https://pulp.plan.io/issues/4898

CHANGES/4898.feature

@@ -0,0 +1 @@
+No duplicated content can be present in a repository version.

pulp_rpm/app/models.py

@@ -4,7 +4,11 @@
 import createrepo_c as cr
 
 from django.contrib.postgres.fields import JSONField
-from django.db import models, transaction
+from django.db import (
+    models,
+    transaction,
+)
+
 from pulpcore.plugin.models import (
     Content,
     ContentArtifact,
@@ -17,6 +21,7 @@
     PublicationDistribution,
     Task,
 )
+from pulpcore.plugin.repo_version_utils import remove_duplicates
 
 from pulp_rpm.app.constants import (CHECKSUM_CHOICES, CR_PACKAGE_ATTRS,
                                     CR_UPDATE_COLLECTION_ATTRS,
@@ -212,6 +217,8 @@ class Package(Content):
     time_build = models.BigIntegerField(null=True)
     time_file = models.BigIntegerField(null=True)
 
+    repo_key_fields = ('name', 'epoch', 'version', 'release', 'arch')
+
     @property
     def filename(self):
         """
@@ -753,6 +760,8 @@ class PackageGroup(Content):
 
     related_packages = models.ManyToManyField(Package)
 
+    repo_key_fields = ('id',)
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
 
@@ -861,6 +870,8 @@ class PackageCategory(Content):
 
     packagegroups = models.ManyToManyField(PackageGroup)
 
+    repo_key_fields = ('id',)
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
 
@@ -973,6 +984,8 @@ class PackageEnvironment(Content):
     packagegroups = models.ManyToManyField(PackageGroup, related_name='packagegroups_to_env')
     optionalgroups = models.ManyToManyField(PackageGroup, related_name='optionalgroups_to_env')
 
+    repo_key_fields = ('id',)
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
 
@@ -1134,6 +1147,21 @@ def new_version(self, base_version=None):
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
 
+    def finalize_new_version(self, new_version):
+        """
+        Ensure there are no duplicates in a repo version and content is not broken.
+
+        Remove duplicates based on repo_key_fields.
+        TODO: Ensure that modulemd is added with all its RPMs.
+        TODO: Ensure that modulemd is removed with all its RPMs.
+        TODO: Resolve advisory conflicts when there is more than one advisory with the same id.
+
+        Args:
+            new_version (pulpcore.app.models.RepositoryVersion): The incomplete RepositoryVersion to
+                finalize.
+        """
+        remove_duplicates(new_version)
+
 
 class RpmRemote(Remote):
     """
@@ -1236,6 +1264,8 @@ class ModulemdDefaults(Content):
 
     digest = models.CharField(unique=True, max_length=64)
 
+    repo_key_fields = ('module',)
+
     @classmethod
     def natural_key_fields(cls):
         """
@@ -1321,6 +1351,15 @@ class DistributionTree(Content):
     discnum = models.IntegerField(null=True)
     totaldiscs = models.IntegerField(null=True)
 
+    repo_key_fields = (
+        "header_version",
+        "release_name",
+        "release_short",
+        "release_version",
+        "arch",
+        "build_timestamp",
+    )
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         unique_together = (
@@ -1544,6 +1583,8 @@ class RepoMetadataFile(Content):
     checksum_type = models.CharField(max_length=6)
     checksum = models.CharField(max_length=128)
 
+    repo_key_fields = ('data_type',)
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         unique_together = ("data_type", "checksum")

pulp_rpm/app/tasks/__init__.py

@@ -1,4 +1,3 @@
 from .publishing import publish  # noqa
 from .synchronizing import synchronize  # noqa
-from .upload import one_shot_upload  # noqa
 from .copy import copy_content  # noqa

pulp_rpm/app/tasks/synchronizing.py

@@ -25,7 +25,6 @@
     DeclarativeContent,
     DeclarativeVersion,
     RemoteArtifactSaver,
-    RemoveDuplicates,
     Stage,
     QueryExistingArtifacts,
     QueryExistingContents
@@ -98,10 +97,6 @@ def synchronize(remote_pk, repository_pk):
     remote = RpmRemote.objects.get(pk=remote_pk)
     repository = RpmRepository.objects.get(pk=repository_pk)
 
-    dupe_criteria = [
-        {'model': Package, 'field_names': ['name', 'epoch', 'version', 'release', 'arch']},
-        {'model': RepoMetadataFile, 'field_names': ['data_type']},
-    ]
     if not remote.url:
         raise ValueError(_('A remote must have a url specified to synchronize.'))
 
@@ -129,14 +124,12 @@ def synchronize(remote_pk, repository_pk):
             if repodata_exists(remote, new_url):
                 stage = RpmFirstStage(remote, deferred_download, new_url=new_url)
                 dv = RpmDeclarativeVersion(first_stage=stage,
-                                           repository=new_repository,
-                                           remove_duplicates=dupe_criteria)
+                                           repository=repository)
                 dv.create()
 
     first_stage = RpmFirstStage(remote, deferred_download, kickstart=kickstart)
     dv = RpmDeclarativeVersion(first_stage=first_stage,
-                               repository=repository,
-                               remove_duplicates=dupe_criteria)
+                               repository=repository)
     dv.create()
 
 
@@ -168,9 +161,6 @@ def pipeline_stages(self, new_version):
             RpmContentSaver(),
             RemoteArtifactSaver(),
         ]
-        for dupe_query_dict in self.remove_duplicates:
-            pipeline.append(RemoveDuplicates(new_version, **dupe_query_dict))
-
         return pipeline
 
 

pulp_rpm/app/viewsets.py

@@ -3,6 +3,7 @@
 from rest_framework.decorators import action
 from rest_framework.parsers import FormParser, MultiPartParser
 
+from pulpcore.plugin.actions import ModifyRepositoryActionMixin
 from pulpcore.plugin.tasking import enqueue_with_reservation
 from pulpcore.plugin.serializers import (
     AsyncOperationResponseSerializer,
@@ -94,7 +95,7 @@ class PackageViewSet(SingleArtifactContentUploadViewSet):
     filterset_class = PackageFilter
 
 
-class RpmRepositoryViewSet(RepositoryViewSet):
+class RpmRepositoryViewSet(RepositoryViewSet, ModifyRepositoryActionMixin):
     """
     A ViewSet for RpmRepository.
     """