Adds copy endpoint and task

* correctly maintains 'deprecated' status from source repo

closes #7621

CHANGES/7621.feature

@@ -0,0 +1,2 @@
+Adds a new ``/pulp/api/v3/ansible/copy/`` endpoint allowing content to be copied from one
+``AnsibleRepository`` version to a destination ``AnsibleRepository``.

MANIFEST.in

@@ -2,3 +2,4 @@ include LICENSE
 include pulp_ansible/app/webserver_snippets/*
 include requirements.txt
 include pyproject.toml
+include pulp_ansible/app/schema/*

pulp_ansible/app/schema/__init__.py

@@ -0,0 +1,7 @@
+import json
+import os
+
+location = os.path.dirname(os.path.realpath(__file__))
+
+with open(os.path.join(location, "copy_config.json")) as copy_config_json:
+    COPY_CONFIG_SCHEMA = json.load(copy_config_json)

pulp_ansible/app/schema/copy_config.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "CopyConfig",
+  "description": "Config for copying content between repos",
+  "type": "array",
+  "minItems": 1,
+  "items": {
+    "type": "object",
+    "additionProperties": false,
+    "required": [ "source_repo_version", "dest_repo" ],
+    "properties": {
+      "source_repo_version": { "type": "string" },
+      "dest_repo": { "type": "string" },
+      "dest_base_version": { "type": "integer" },
+      "content": {
+        "type": "array",
+        "items": { "type": "string" }
+      }
+    }
+  }
+}

pulp_ansible/app/serializers.py

@@ -1,6 +1,7 @@
 from gettext import gettext as _
 
 from django.conf import settings
+from jsonschema import Draft7Validator
 from rest_framework import serializers
 
 from pulpcore.plugin.serializers import (
@@ -10,6 +11,7 @@
     RepositorySerializer,
     SingleArtifactContentSerializer,
     RepositoryVersionDistributionSerializer,
+    validate_unknown_fields,
 )
 
 from .models import (
@@ -23,6 +25,7 @@
     Role,
     Tag,
 )
+from pulp_ansible.app.schema import COPY_CONFIG_SCHEMA
 from pulp_ansible.app.tasks.utils import parse_collections_requirements_file
 
 
@@ -374,20 +377,15 @@ class CopySerializer(serializers.Serializer):
     """
 
     config = serializers.JSONField(
-        help_text=_(
-            "A JSON document describing sources, destinations, and content to be copied"
-        ),
-    )
-
-    dependency_solving = serializers.BooleanField(
-        help_text=_("Also copy dependencies of the content being copied."), default=True
+        help_text=_("A JSON document describing sources, destinations, and content to be copied"),
     )
 
     def validate(self, data):
         """
         Validate that the Serializer contains valid data.
-        Set the RpmRepository based on the RepositoryVersion if only the latter is provided.
-        Set the RepositoryVersion based on the RpmRepository if only the latter is provided.
+
+        Set the AnsibleRepository based on the RepositoryVersion if only the latter is provided.
+        Set the RepositoryVersion based on the AnsibleRepository if only the latter is provided.
         Convert the human-friendly names of the content types into what Pulp needs to query on.
         """
         super().validate(data)

pulp_ansible/app/tasks/copy.py

@@ -1,20 +1,25 @@
 from django.db import transaction
+from django.db.models import Q
+from pulpcore.plugin.models import RepositoryVersion
+
+from pulp_ansible.app.models import AnsibleCollectionDeprecated, AnsibleRepository, Collection
 
 
 @transaction.atomic
-def copy_content(config, dependency_solving):
+def copy_content(config):
     """
     Copy content from one repo to another.
-    Args:
-        source_repo_version_pk: repository version primary key to copy units from
-        dest_repo_pk: repository primary key to copy units into
-        criteria: a dict that maps type to a list of criteria to filter content by. Note that this
-            criteria MUST be validated before being passed to this task.
-        content_pks: a list of content pks to copy from source to destination
+
+    Accepts a config containing:
+      * source_repo_version_pk: repository version primary key to copy units from
+      * dest_repo_pk: repository primary key to copy units into
+      * content_pks: a list of content pks to copy from source to destination
+
     """
+
     def process_entry(entry):
         source_repo_version = RepositoryVersion.objects.get(pk=entry["source_repo_version"])
-        dest_repo = RpmRepository.objects.get(pk=entry["dest_repo"])
+        dest_repo = AnsibleRepository.objects.get(pk=entry["dest_repo"])
 
         dest_version_provided = bool(entry.get("dest_base_version"))
         if dest_version_provided:
@@ -28,65 +33,37 @@ def process_entry(entry):
             content_filter = Q()
 
         return (
-            source_repo_version, dest_repo_version,
-            dest_repo, content_filter, dest_version_provided
+            source_repo_version,
+            dest_repo_version,
+            dest_repo,
+            content_filter,
+            dest_version_provided,
         )
 
-    if not dependency_solving:
-        # No Dependency Solving Branch
-        # ============================
-        for entry in config:
-            (source_repo_version, dest_repo_version,
-                dest_repo, content_filter, dest_version_provided) = process_entry(entry)
-
-            content_to_copy = source_repo_version.content.filter(content_filter)
-            content_to_copy |= find_children_of_content(content_to_copy, source_repo_version)
-
-            base_version = dest_repo_version if dest_version_provided else None
-            with dest_repo.new_version(base_version=base_version) as new_version:
-                new_version.add_content(content_to_copy)
-    else:
-        # Dependency Solving Branch
-        # =========================
-
-        # TODO: a more structured way to store this state would be nice.
-        content_to_copy = {}
-        repo_mapping = {}
-        libsolv_repo_names = {}
-        base_versions = {}
-
-        solver = Solver()
-
-        for entry in config:
-            (source_repo_version, dest_repo_version,
-                dest_repo, content_filter, dest_version_provided) = process_entry(entry)
-
-            repo_mapping[source_repo_version] = dest_repo_version
-            base_versions[source_repo_version] = dest_version_provided
-
-            # Load the content from the source and destination repository versions into the solver
-            source_repo_name = solver.load_source_repo(source_repo_version)
-            solver.load_target_repo(dest_repo_version)
-
-            # Store the correspondance between the libsolv name of a repo version and the
-            # actual Pulp repo version, so that we can work backwards to get the latter
-            # from the former.
-            libsolv_repo_names[source_repo_name] = source_repo_version
-
-            # Find all of the matching content in the repository version, then determine
-            # child relationships (e.g. RPM children of Errata/Advisories), then combine
-            # those two sets to copy the specified content + children.
-            content = source_repo_version.content.filter(content_filter)
-            children = find_children_of_content(content, source_repo_version)
-            content_to_copy[source_repo_name] = content | children
-
-        solver.finalize()
-
-        content_to_copy = solver.resolve_dependencies(content_to_copy)
-
-        for from_repo, units in content_to_copy.items():
-            src_repo_version = libsolv_repo_names[from_repo]
-            dest_repo_version = repo_mapping[src_repo_version]
-            base_version = dest_repo_version if base_versions[src_repo_version] else None
-            with dest_repo_version.repository.new_version(base_version=base_version) as new_version:
-                new_version.add_content(Content.objects.filter(pk__in=units))
+    for entry in config:
+        (
+            source_repo_version,
+            dest_repo_version,
+            dest_repo,
+            content_filter,
+            dest_version_provided,
+        ) = process_entry(entry)
+
+        content_to_copy = source_repo_version.content.filter(content_filter)
+
+        base_version = dest_repo_version if dest_version_provided else None
+        with dest_repo.new_version(base_version=base_version) as new_version:
+            new_version.add_content(content_to_copy)
+            deprecated_in_source_repo_version_qs = Collection.objects.filter(
+                ansiblecollectiondeprecated__repository_version=source_repo_version
+            )
+            to_deprecate = []
+            for collection_deprecated in deprecated_in_source_repo_version_qs:
+                to_deprecate.append(
+                    AnsibleCollectionDeprecated(
+                        repository_version=new_version, collection=collection_deprecated
+                    )
+                )
+
+            if to_deprecate:
+                AnsibleCollectionDeprecated.objects.bulk_create(to_deprecate, ignore_conflicts=True)

pulp_ansible/app/urls.py

@@ -1,4 +1,5 @@
 from django.conf import settings
+from django.conf.urls import url
 from django.urls import include, path
 
 from pulp_ansible.app.galaxy.views import (
@@ -12,7 +13,7 @@
 )
 from pulp_ansible.app.galaxy.v3 import views as views_v3
 
-from pulp_ansible.app.viewsets import CollectionUploadViewSet
+from pulp_ansible.app.viewsets import CopyViewSet, CollectionUploadViewSet
 
 
 GALAXY_API_ROOT = getattr(settings, "GALAXY_API_ROOT", "pulp_ansible/galaxy/<path:path>/api/")
@@ -79,4 +80,5 @@
     path(GALAXY_API_ROOT + "v1/", include(v1_urls)),
     path(GALAXY_API_ROOT + "v2/", include(v2_urls)),
     path(GALAXY_API_ROOT + "v3/", include(v3_urls)),
+    url(r"^pulp/api/v3/ansible/copy/$", CopyViewSet.as_view({"post": "create"})),
 ]

pulp_ansible/app/viewsets.py

@@ -12,9 +12,10 @@
 from rest_framework.decorators import action
 from rest_framework.filters import OrderingFilter
 from rest_framework.parsers import FormParser, MultiPartParser
+from rest_framework.serializers import ValidationError as DRFValidationError
 
 from pulpcore.plugin.exceptions import DigestValidationError
-from pulpcore.plugin.models import PulpTemporaryFile
+from pulpcore.plugin.models import PulpTemporaryFile, RepositoryVersion
 from pulpcore.plugin.serializers import (
     AsyncOperationResponseSerializer,
     RepositorySyncURLSerializer,
@@ -339,33 +340,29 @@ class CopyViewSet(viewsets.ViewSet):
     serializer_class = CopySerializer
 
     @extend_schema(
-        description="Trigger an asynchronous task to copy ansible content from one repository "\
-                    "into another, creating a new repository version.",
+        description="Trigger an asynchronous task to copy ansible content from one repository "
+        "into another, creating a new repository version.",
         summary="Copy content",
         operation_id="copy_content",
         request=CopySerializer,
-        responses={202: AsyncOperationResponseSerializer}
+        responses={202: AsyncOperationResponseSerializer},
     )
     def create(self, request):
         """Copy content."""
-        serializer = CopySerializer(data=request.data, context={'request': request})
+        serializer = CopySerializer(data=request.data, context={"request": request})
         serializer.is_valid(raise_exception=True)
 
-        dependency_solving = serializer.validated_data['dependency_solving']
-        config = serializer.validated_data['config']
+        config = serializer.validated_data["config"]
 
         config, repos = self._process_config(config)
 
-        async_result = enqueue_with_reservation(
-            tasks.copy_content, repos,
-            args=[config, dependency_solving],
-            kwargs={}
-        )
+        async_result = enqueue_with_reservation(copy_content, repos, args=[config], kwargs={})
         return OperationPostponedResponse(async_result, request)
 
     def _process_config(self, config):
         """
         Change the hrefs into pks within config.
+
         This method also implicitly validates that the hrefs map to objects and it returns a list of
         repos so that the task can lock on them.
         """
@@ -374,21 +371,23 @@ def _process_config(self, config):
 
         for entry in config:
             r = dict()
-            source_version = NamedModelViewSet().get_resource(entry["source_repo_version"],
-                                                              RepositoryVersion)
-            dest_repo = NamedModelViewSet().get_resource(entry["dest_repo"], RpmRepository)
+            source_version = NamedModelViewSet().get_resource(
+                entry["source_repo_version"], RepositoryVersion
+            )
+            dest_repo = NamedModelViewSet().get_resource(entry["dest_repo"], AnsibleRepository)
             r["source_repo_version"] = source_version.pk
             r["dest_repo"] = dest_repo.pk
             repos.extend((source_version.repository, dest_repo))
 
             if "dest_base_version" in entry:
                 try:
-                    r["dest_base_version"] = dest_repo.versions.\
-                        get(number=entry["dest_base_version"]).pk
+                    r["dest_base_version"] = dest_repo.versions.get(
+                        number=entry["dest_base_version"]
+                    ).pk
                 except RepositoryVersion.DoesNotExist:
-                    message = _("Version {version} does not exist for repository "
-                                "'{repo}'.").format(version=entry["dest_base_version"],
-                                                    repo=dest_repo.name)
+                    message = _(
+                        "Version {version} does not exist for repository " "'{repo}'."
+                    ).format(version=entry["dest_base_version"], repo=dest_repo.name)
                     raise DRFValidationError(detail=message)
 
             if entry.get("content") is not None:

requirements.txt

@@ -1,4 +1,5 @@
 galaxy_importer
+jsonschema>=3.0
 packaging
 pulpcore>=3.7
 PyYAML