Content contained in RepoVer and Publication

Views to show repository_versions or publications which contains
specific content.

closes: #4832
https://pulp.plan.io/issues/4832

CHANGES/4832.feature

@@ -0,0 +1 @@
+Added two views to identify content which belongs to repository_version or publication.

CHANGES/plugin_api/4832.deprecation

@@ -0,0 +1 @@
+RepositoryVersion method "versions_containing_content" is deprecated now.

pulpcore/app/management/commands/handle-artifact-checksums.py

@@ -66,9 +66,7 @@ def _show_on_demand_content(self, checksums):
 
         content_artifacts = ContentArtifact.objects.filter(remoteartifact__pk__in=remote_artifacts)
         content = Content.objects.filter(contentartifact__pk__in=content_artifacts)
-        repo_versions = RepositoryVersion.versions_containing_content(content).select_related(
-            "repository"
-        )
+        repo_versions = RepositoryVersion.objects.with_content(content).select_related("repository")
 
         self.stdout.write(
             _("Found {} on-demand content units with forbidden checksums.").format(content.count())
@@ -99,9 +97,7 @@ def _show_immediate_content(self, forbidden_checksums):
         artifacts = Artifact.objects.filter(query_forbidden | query_required)
         content_artifacts = ContentArtifact.objects.filter(artifact__in=artifacts)
         content = Content.objects.filter(contentartifact__pk__in=content_artifacts)
-        repo_versions = RepositoryVersion.versions_containing_content(content).select_related(
-            "repository"
-        )
+        repo_versions = RepositoryVersion.objects.with_content(content).select_related("repository")
 
         self.stdout.write(
             _("Found {} downloaded content units with forbidden or missing checksums.").format(

pulpcore/app/models/publication.py

@@ -10,6 +10,30 @@
 from pulpcore.app.loggers import deprecation_logger
 
 
+class PublicationQuerySet(models.QuerySet):
+    """A queryset that provides publication filtering methods."""
+
+    def with_content(self, content):
+        """
+        Filters publictions that contain the provided content units.
+
+        Args:
+            content (django.db.models.QuerySet): query of content
+
+        Returns:
+            django.db.models.QuerySet: Publications that contain content.
+        """
+        pub_artifact_q = Publication.objects.filter(
+            published_artifact__content_artifact__content__pk__in=content
+        )
+        pass_thru_q = Publication.objects.filter(
+            pass_through=True,
+            repository_version__pk__in=RepositoryVersion.objects.with_content(content),
+        )
+
+        return pub_artifact_q | pass_thru_q
+
+
 class Publication(MasterModel):
     """
     A publication contains metadata and artifacts associated with content
@@ -44,6 +68,8 @@ class Publication(MasterModel):
 
     TYPE = "publication"
 
+    objects = PublicationQuerySet.as_manager()
+
     complete = models.BooleanField(db_index=True, default=False)
     pass_through = models.BooleanField(default=False)
 

pulpcore/app/models/repository.py

@@ -19,6 +19,7 @@
 from pulpcore.constants import ALL_KNOWN_CONTENT_CHECKSUMS
 from pulpcore.download.factory import DownloaderFactory
 from pulpcore.exceptions import ResourceImmutableError
+from pulpcore.app.loggers import deprecation_logger
 
 from .base import MasterModel, BaseModel
 from .content import Artifact, Content
@@ -465,6 +466,31 @@ class RepositoryVersionQuerySet(models.QuerySet):
     def complete(self):
         return self.exclude(complete=False)
 
+    def with_content(self, content):
+        """
+        Filters repository versions that contain the provided content units.
+
+        Args:
+            content (django.db.models.QuerySet): query of content
+
+        Returns:
+            django.db.models.QuerySet: Repository versions which contains content.
+        """
+        query = models.Q(pk__in=[])
+        repo_content = RepositoryContent.objects.filter(content__pk__in=content)
+
+        for rc in repo_content.iterator():
+            filter = models.Q(
+                repository__pk=rc.repository.pk,
+                number__gte=rc.version_added.number,
+            )
+            if rc.version_removed:
+                filter &= models.Q(number__lt=rc.version_removed.number)
+
+            query |= filter
+
+        return self.filter(query)
+
 
 class RepositoryVersion(BaseModel):
     """
@@ -494,6 +520,8 @@ class RepositoryVersion(BaseModel):
         repository (models.ForeignKey): The associated repository.
     """
 
+    objects = RepositoryVersionQuerySet.as_manager()
+
     repository = models.ForeignKey(Repository, on_delete=models.CASCADE)
     number = models.PositiveIntegerField(db_index=True)
     complete = models.BooleanField(db_index=True, default=False)
@@ -518,6 +546,10 @@ def versions_containing_content(cls, content):
         Returns:
             django.db.models.QuerySet: Repository versions which contains content.
         """
+        deprecation_logger(
+            "This method is deprecated and will be removed in version 3.14. "
+            "Use RepositoryVersion.objects.with_content() instead."
+        )
         query = models.Q(pk__in=[])
         repo_content = RepositoryContent.objects.filter(content__pk__in=content)
 

pulpcore/app/urls.py

@@ -16,6 +16,7 @@
 
 from pulpcore.app.apps import pulp_plugin_configs
 from pulpcore.app.views import OrphansView, PulpImporterImportCheckView, RepairView, StatusView
+from pulpcore.app.viewsets import ListRepositoryVersionViewSet
 from pulpcore.constants import API_ROOT
 from pulpcore.openapi import PulpSchemaGenerator
 
@@ -123,6 +124,10 @@ def __repr__(self):
     url(r"^{api_root}repair/".format(api_root=API_ROOT), RepairView.as_view()),
     url(r"^{api_root}status/".format(api_root=API_ROOT), StatusView.as_view()),
     url(r"^{api_root}orphans/".format(api_root=API_ROOT), OrphansView.as_view()),
+    url(
+        r"^{api_root}repository_versions/".format(api_root=API_ROOT),
+        ListRepositoryVersionViewSet.as_view({"get": "list"}),
+    ),
     url(
         r"^{api_root}importers/core/pulp/import-check/".format(api_root=API_ROOT),
         PulpImporterImportCheckView.as_view(),

pulpcore/app/viewsets/__init__.py

@@ -42,6 +42,7 @@
     DistributionFilter,
     DistributionViewSet,
     ListContentGuardViewSet,
+    ListPublicationViewSet,
     NewDistributionFilter,
     PublicationFilter,
     PublicationViewSet,
@@ -54,6 +55,7 @@
     RemoteViewSet,
     RepositoryViewSet,
     RepositoryVersionViewSet,
+    ListRepositoryVersionViewSet,
 )
 from .task import TaskViewSet, TaskGroupViewSet, WorkerViewSet  # noqa
 from .upload import UploadViewSet  # noqa

pulpcore/app/viewsets/publication.py

@@ -1,10 +1,17 @@
 from gettext import gettext as _
 
+from django_filters import Filter
 from django_filters.rest_framework import DjangoFilterBackend, filters
-from rest_framework import mixins
+from rest_framework import mixins, serializers
 from rest_framework.filters import OrderingFilter
 
-from pulpcore.app.models import BaseDistribution, ContentGuard, Distribution, Publication
+from pulpcore.app.models import (
+    BaseDistribution,
+    ContentGuard,
+    Distribution,
+    Publication,
+    Content,
+)
 from pulpcore.app.serializers import (
     BaseDistributionSerializer,
     ContentGuardSerializer,
@@ -27,9 +34,30 @@
 from pulpcore.app.loggers import deprecation_logger
 
 
+class PublicationContentFilter(Filter):
+    def __init__(self, *args, **kwargs):
+        kwargs.setdefault("help_text", _("Content Unit referenced by HREF"))
+        super().__init__(*args, **kwargs)
+
+    def filter(self, qs, value):
+        if value is None:
+            # user didn't supply a value
+            return qs
+
+        if not value:
+            raise serializers.ValidationError(detail=_("No value supplied for content filter"))
+
+        # Get the content object from the content_href
+        content = NamedModelViewSet.get_resource(value, Content)
+
+        return qs.with_content([content.pk])
+
+
 class PublicationFilter(BaseFilterSet):
     repository_version = RepositoryVersionFilter()
     pulp_created = IsoDateTimeFilter()
+    content = PublicationContentFilter()
+    content__in = PublicationContentFilter(field_name="content", lookup_expr="in")
 
     class Meta:
         model = Publication
@@ -39,6 +67,18 @@ class Meta:
         }
 
 
+class ListPublicationViewSet(NamedModelViewSet, mixins.ListModelMixin):
+    endpoint_name = "publications"
+    queryset = Publication.objects.all()
+    serializer_class = PublicationSerializer
+    filterset_class = PublicationFilter
+
+    @classmethod
+    def is_master_viewset(cls):
+        """Do not hide from the routers."""
+        return False
+
+
 class PublicationViewSet(
     NamedModelViewSet, mixins.RetrieveModelMixin, mixins.ListModelMixin, mixins.DestroyModelMixin
 ):

pulpcore/app/viewsets/repository.py

@@ -1,4 +1,3 @@
-import itertools
 from gettext import gettext as _
 
 from django_filters import Filter
@@ -7,13 +6,13 @@
 from rest_framework import mixins, serializers
 from rest_framework.decorators import action
 from rest_framework.filters import OrderingFilter
+from rest_framework.viewsets import GenericViewSet
 
 from pulpcore.app import tasks
 from pulpcore.app.models import (
     Content,
     Remote,
     Repository,
-    RepositoryContent,
     RepositoryVersion,
 )
 from pulpcore.app.response import OperationPostponedResponse
@@ -100,12 +99,6 @@ class RepositoryViewSet(ImmutableRepositoryViewSet, AsyncUpdateMixin):
 class RepositoryVersionContentFilter(Filter):
     """
     Filter used to get the repository versions where some given content can be found.
-
-    Given a content_href, this filter will:
-        1. Get the RepositoryContent that the content can be found in
-        2. Get a list of version_added and version_removed where the content was
-           changed on the repository
-        3. Calculate and return the versions that the content can be found on
     """
 
     def __init__(self, *args, **kwargs):
@@ -132,46 +125,7 @@ def filter(self, qs, value):
         # Get the content object from the content_href
         content = NamedModelViewSet.get_resource(value, Content)
 
-        # Get the repository from the parent request.
-        repository_pk = self.parent.request.parser_context["kwargs"]["repository_pk"]
-        repository = Repository.objects.get(pk=repository_pk)
-
-        repository_content_set = RepositoryContent.objects.filter(
-            content=content, repository=repository
-        )
-
-        # Get the sorted list of version_added and version_removed.
-        version_added = list(repository_content_set.values_list("version_added__number", flat=True))
-
-        # None values have to be filtered out from version_removed,
-        # in order for zip_longest to pass it a default fillvalue
-        version_removed = list(
-            filter(
-                None.__ne__,
-                repository_content_set.values_list("version_removed__number", flat=True),
-            )
-        )
-
-        # The range finding should work as long as both lists are sorted
-        # Why it works: https://gist.github.com/werwty/6867f83ae5adbae71e452c28ecd9c444
-        version_added.sort()
-        version_removed.sort()
-
-        # Match every version_added to a version_removed, if len(version_removed)
-        # is shorter than len(version_added), pad out the remaining space with the current
-        # repository version +1 (the +1 is to the current version gets included when we
-        # calculate range)
-        version_tuples = itertools.zip_longest(
-            version_added, version_removed, fillvalue=repository.next_version
-        )
-
-        # Get the ranges between paired version_added and version_removed to get all
-        # the versions the content is present in.
-        versions = [list(range(added, removed)) for (added, removed) in version_tuples]
-        # Flatten the list of lists
-        versions = list(itertools.chain.from_iterable(versions))
-
-        return qs.filter(number__in=versions)
+        return qs.with_content([content.pk])
 
 
 class RepositoryVersionFilter(BaseFilterSet):
@@ -278,3 +232,15 @@ class RemoteViewSet(
     serializer_class = RemoteSerializer
     queryset = Remote.objects.all()
     filterset_class = RemoteFilter
+
+
+# We have to use GenericViewSet as NamedModelViewSet causes
+# get_viewset_for_model() to match multiple viewsets.
+class ListRepositoryVersionViewSet(
+    GenericViewSet,
+    mixins.ListModelMixin,
+):
+    endpoint_name = "repository_versions"
+    serializer_class = RepositoryVersionSerializer
+    queryset = RepositoryVersion.objects.all()
+    filterset_class = RepositoryVersionFilter

pulpcore/tests/functional/api/using_plugin/constants.py

@@ -5,12 +5,17 @@
 from pulp_smash import config
 from pulp_smash.pulp3.constants import (
     BASE_DISTRIBUTION_PATH,
+    BASE_PATH,
     BASE_PUBLICATION_PATH,
     BASE_REMOTE_PATH,
     BASE_REPO_PATH,
     BASE_CONTENT_PATH,
 )
 
+PULP_REPOSITORY_VERSION_BASE_URL = urljoin(BASE_PATH, "repository_versions/")
+
+PULP_PUBLICATION_BASE_URL = urljoin(BASE_PATH, "publications/")
+
 PULP_FIXTURES_BASE_URL = config.get_config().get_fixtures_url()
 
 PULP_CONTENT_HOST_BASE_URL = config.get_config().get_content_host_base_url()