Initial copy implementation

Criteria and depsolving are no-op -- copies everything

Also includes a basic smash test

re: #6018
https://pulp.plan.io/issues/6018

pulp_rpm/app/serializers.py

@@ -6,22 +6,19 @@
 from rest_framework import serializers
 from rest_framework.exceptions import NotAcceptable
 
-from pulpcore.plugin.models import (
-    Remote,
-    RepositoryVersion,
-)
+from pulpcore.plugin.models import Remote
 from pulpcore.plugin.serializers import (
     ArtifactSerializer,
     ContentChecksumSerializer,
     DetailRelatedField,
     ModelSerializer,
     MultipleArtifactContentSerializer,
-    NestedRelatedField,
     NoArtifactContentSerializer,
     PublicationDistributionSerializer,
     PublicationSerializer,
     RepositorySyncURLSerializer,
     RelatedField,
+    DetailRelatedField,
     RemoteSerializer,
     RepositorySerializer,
     SingleArtifactContentUploadSerializer,
@@ -40,7 +37,6 @@
     PULP_UPDATE_COLLECTION_ATTRS,
     PULP_UPDATE_RECORD_ATTRS,
     PULP_UPDATE_REFERENCE_ATTRS,
-    RPM_PLUGIN_TYPE_CHOICE_MAP,
     SKIP_TYPES
 )
 
@@ -589,36 +585,51 @@ class RpmRepositorySyncURLSerializer(RepositorySyncURLSerializer):
         )
     )
 
+# SCHEMA = '''{
+#     "$schema": "http://json-schema.org/draft-07/schema#",
+#     "title": "CopyPlan",
+#     "type": "object",
+#     "properties": {
+#         "type": "object",
+#         "minItems": 1,
+#         "additionalProperties": false
+#         }
+#     },
+#     "additionalProperties": false
+# }'''
+
 
 class CopySerializer(serializers.Serializer):
     """
     A serializer for Content Copy API.
     """
 
-    source_repo = serializers.HyperlinkedRelatedField(
-        help_text=_('A URI of the repository.'),
-        required=False,
+    source_repo = DetailRelatedField(
+        help_text=_('A URI of the repository to copy from.'),
         queryset=RpmRepository.objects.all(),
-        view_name='repositories-rpm/rpm-detail',
     )
-    source_repo_version = NestedRelatedField(
-        help_text=_('A URI of the repository version'),
-        required=False,
-        queryset=RepositoryVersion.objects.all(),
-        parent_lookup_kwargs={'repository_pk': 'repository__pk'},
-        lookup_field='number',
-        view_name='versions-detail',
-    )
-    dest_repo = serializers.HyperlinkedRelatedField(
-        help_text=_('A URI of the repository.'),
-        required=True,
+
+    dest_repo = DetailRelatedField(
+        help_text=_('A URI of the repository to copy to.'),
         queryset=RpmRepository.objects.all(),
-        view_name='repositories-rpm/rpm-detail',
     )
-    types = serializers.ListField(
-        help_text=_('A list of types to copy ["package", "advisory"]'),
-        write_only=True,
-        default=['package', 'advisory']
+
+    # source_repo_version = NestedRelatedField(
+    #     help_text=_('A URI of the repository version'),
+    #     required=False,
+    #     queryset=RepositoryVersion.objects.all(),
+    #     parent_lookup_kwargs={'repository_pk': 'repository__pk'},
+    #     lookup_field='number',
+    #     view_name='versions-detail',
+    # )
+
+    criteria = serializers.JSONField(
+        help_text=_('A JSON document describing what content you want to be copied'),
+    )
+
+    dependency_solving = serializers.BooleanField(
+        help_text=_('Also copy dependencies of the content being copied.'),
+        default=True
     )
 
     def validate(self, data):
@@ -631,55 +642,55 @@ def validate(self, data):
 
         """
         super().validate(data)
+
+        # schema = json.loads(SCHEMA)
+        # validator = Draft7Validator(schema)
+
         if hasattr(self, 'initial_data'):
             validate_unknown_fields(self.initial_data, self.fields)
 
-        new_data = {}
-        new_data.update(data)
+        # criteria = data.get('criteria')
 
-        source_repo = data.get('source_repo')
-        source_repo_version = data.get('source_repo_version')
+        # err = []
+        # for error in sorted(validator.iter_errors(criteria), key=str):
+        #     err.append(error.message)
+        # if err:
+        #     raise serializers.ValidationError(
+        #         _("Provided copy criteria is invalid:'{}'".format(err))
+        #     )
 
-        if not source_repo and not source_repo_version:
-            raise serializers.ValidationError(
-                _("Either the 'source_repo' or 'source_repo_version' need to be specified"))
+        return data
 
-        if source_repo and source_repo_version:
-            raise serializers.ValidationError(
-                _("Either the 'source_repo' or 'source_repo_version' need to be specified "
-                  "but not both.")
-            )
+        # new_data = {}
+        # new_data.update(data)
+
+        # source_repo = data.get('source_repo')
+        # source_repo_version = data.get('source_repo_version')
+
+        # if not source_repo and not source_repo_version:
+        #     raise serializers.ValidationError(
+        #         _("Either the 'source_repo' or 'source_repo_version' need to be specified"))
+
+        # if source_repo and source_repo_version:
+        #     raise serializers.ValidationError(
+        #         _("Either the 'source_repo' or 'source_repo_version' need to be specified "
+        #           "but not both.")
+        #     )
+
+        # if not source_repo and source_repo_version:
+        #     repo = {'source_repo': source_repo_version.repository}
+        #     new_data.update(repo)
+
+        # if source_repo and not source_repo_version:
+        #     version = RepositoryVersion.latest(source_repo)
+        #     if version:
+        #         repo_version = {'source_repo_version': version}
+        #         new_data.update(repo_version)
+        #     else:
+        #         raise serializers.ValidationError(
+        #             detail=_('Repository has no version available to copy'))
 
-        if not source_repo and source_repo_version:
-            repo = {'source_repo': source_repo_version.repository}
-            new_data.update(repo)
-
-        if source_repo and not source_repo_version:
-            version = RepositoryVersion.latest(source_repo)
-            if version:
-                repo_version = {'source_repo_version': version}
-                new_data.update(repo_version)
-            else:
-                raise serializers.ValidationError(
-                    detail=_('Repository has no version available to copy'))
-
-        types = data.get('types')
-        final_types = []
-
-        if types:
-            for t in types:
-                substitution = RPM_PLUGIN_TYPE_CHOICE_MAP.get(t)
-                if not substitution:
-                    raise serializers.ValidationError(_(
-                        "'{type}' is an invalid type, please use one of {choices}".format(
-                            type=t,
-                            choices=list(RPM_PLUGIN_TYPE_CHOICE_MAP.keys())
-                        ))
-                    )
-                final_types.append(substitution)
-            new_data.update({'types': final_types})
-
-        return new_data
+        # return new_data
 
 
 class PackageGroupSerializer(NoArtifactContentSerializer):

pulp_rpm/app/tasks/copy.py

@@ -2,8 +2,10 @@
 
 from pulpcore.plugin.models import Repository, RepositoryVersion
 
+from pulp_rpm.app.models import RpmRepository
 
-def copy_content(source_repo_version_pk, dest_repo_pk, types):
+
+def copy_content(source_repo_version_pk, dest_repo_pk, criteria, dependency_solving):
     """
     Copy content from one repo to another.
 
@@ -13,15 +15,13 @@ def copy_content(source_repo_version_pk, dest_repo_pk, types):
         types: a tuple of strings representing the '_type' values of types to include in the copy
     """
     source_repo_version = RepositoryVersion.objects.get(pk=source_repo_version_pk)
-    dest_repo = Repository.objects.get(pk=dest_repo_pk)
+    source_repo = RpmRepository.objects.get(pk=source_repo_version.repository)
+    content_types = source_repo.CONTENT_TYPES
+    list(content_types)
+
+    dest_repo = RpmRepository.objects.get(pk=dest_repo_pk)
 
-    query = None
-    for ptype in types:
-        if query:
-            query = query | Q(_type=ptype)
-        else:
-            query = Q(_type=ptype)
+    content_to_copy = source_repo_version.content
 
-    content_to_copy = source_repo_version.content.filter(query)
-    with RepositoryVersion.create(dest_repo) as new_version:
+    with dest_repo.new_version() as new_version:
         new_version.add_content(content_to_copy)

pulp_rpm/app/urls.py

@@ -0,0 +1,8 @@
+from django.conf.urls import url
+
+from .viewsets import CopyViewSet
+
+
+urlpatterns = [
+    url(r'^pulp/api/v3/rpm/copy/$', CopyViewSet.as_view({'post': 'create'}))
+]

pulp_rpm/app/viewsets.py

@@ -2,7 +2,6 @@
 from drf_yasg.utils import swagger_auto_schema
 from rest_framework import viewsets, mixins
 from rest_framework.decorators import action
-from rest_framework.parsers import FormParser, MultiPartParser
 
 from pulpcore.plugin.actions import ModifyRepositoryActionMixin
 from pulpcore.plugin.tasking import enqueue_with_reservation
@@ -238,7 +237,6 @@ class CopyViewSet(viewsets.ViewSet):
     """
 
     serializer_class = CopySerializer
-    parser_classes = (MultiPartParser, FormParser)
 
     @swagger_auto_schema(
         operation_description="Trigger an asynchronous task to copy RPM content"
@@ -255,13 +253,17 @@ def create(self, request):
         serializer.is_valid(raise_exception=True)
 
         source_repo = serializer.validated_data['source_repo']
-        source_repo_version = serializer.validated_data['source_repo_version']
+        # source_repo_version = serializer.validated_data['source_repo_version']
         dest_repo = serializer.validated_data['dest_repo']
-        types = serializer.validated_data['types']
+        criteria = serializer.validated_data['criteria']
+        dependency_solving = serializer.validated_data['dependency_solving']
+
+        source_repos = [source_repo]
+        dest_repos = [dest_repo]
 
         async_result = enqueue_with_reservation(
-            tasks.copy_content, [source_repo, dest_repo],
-            args=[source_repo_version.pk, dest_repo.pk, types],
+            tasks.copy_content, [*source_repos, *dest_repos],
+            args=[source_repo.latest_version().pk, dest_repo.pk, criteria, dependency_solving],
             kwargs={}
         )
         return OperationPostponedResponse(async_result, request)

pulp_rpm/tests/functional/api/test_copy.py

@@ -0,0 +1,91 @@
+# coding=utf-8
+"""Tests that sync rpm plugin repositories."""
+import unittest
+
+from pulp_smash import api, config
+from pulp_smash.pulp3.utils import (
+    delete_orphans,
+    gen_repo,
+    get_added_content_summary,
+    get_content_summary,
+    get_removed_content,
+    sync,
+)
+
+from pulp_rpm.tests.functional.constants import (
+    RPM_ADVISORY_CONTENT_NAME,
+    RPM_FIXTURE_SUMMARY,
+    RPM_REMOTE_PATH,
+    RPM_REPO_PATH,
+)
+from pulp_rpm.tests.functional.utils import gen_rpm_remote, rpm_copy
+from pulp_rpm.tests.functional.utils import set_up_module as setUpModule  # noqa:F401
+
+
+class BasicCopyTestCase(unittest.TestCase):
+    """Sync repositories with the rpm plugin."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Create class-wide variables."""
+        cls.cfg = config.get_config()
+        cls.client = api.Client(cls.cfg, api.json_handler)
+
+        delete_orphans(cls.cfg)
+
+    def _do_test(self, criteria, expected_results):
+        """Test copying content units with the RPM plugin.
+
+        Do the following:
+
+        1. Create two repositories and a remote.
+        2. Sync the remote.
+        3. Assert that repository version is not None.
+        4. Assert that the correct number of units were added and are present in the repo.
+        5. Use the RPM copy API to units from the repo to the empty repo.
+        7. Assert that the correct number of units were added and are present in the dest repo.
+        """
+        source_repo = self.client.post(RPM_REPO_PATH, gen_repo())
+        self.addCleanup(self.client.delete, source_repo['pulp_href'])
+
+        dest_repo = self.client.post(RPM_REPO_PATH, gen_repo())
+        self.addCleanup(self.client.delete, dest_repo['pulp_href'])
+
+        # Create a remote with the standard test fixture url.
+        body = gen_rpm_remote()
+        remote = self.client.post(RPM_REMOTE_PATH, body)
+        self.addCleanup(self.client.delete, remote['pulp_href'])
+
+        # Sync the repository.
+        self.assertEqual(source_repo["latest_version_href"],
+                         f"{source_repo['pulp_href']}versions/0/")
+        sync(self.cfg, remote, source_repo)
+        source_repo = self.client.get(source_repo['pulp_href'])
+
+        # Check that we have the correct content counts.
+        self.assertDictEqual(get_content_summary(source_repo), RPM_FIXTURE_SUMMARY)
+        self.assertDictEqual(
+            get_added_content_summary(source_repo), RPM_FIXTURE_SUMMARY
+        )
+
+        # Copy all RPMs
+        criteria = {}
+        rpm_copy(self.cfg, source_repo, dest_repo, criteria)
+        dest_repo = self.client.get(source_repo['pulp_href'])
+
+        # Check that we have the correct content counts.
+        self.assertDictEqual(get_content_summary(dest_repo), expected_results)
+        self.assertDictEqual(
+            get_added_content_summary(dest_repo), expected_results,
+        )
+
+    def test_copy_all(self):
+        """Test copying all the content from one repo to another."""
+        criteria = {}
+        results = RPM_FIXTURE_SUMMARY
+        self._do_test(criteria, results)
+
+    # def test_copy_by_href(self):
+    #     criteria = {}
+    #     results = {}
+    #     self._do_test(criteria, results)

pulp_rpm/tests/functional/constants.py

@@ -12,10 +12,10 @@
     BASE_REMOTE_PATH,
 )
 
-PULP_FIXTURES_BASE_URL = config.get_config().get_fixtures_url()
+RPM_COPY_PATH = urljoin(BASE_PATH, 'rpm/copy/')
+"""The URL used for copying RPM content between repos."""
 
-DRPM_UNSIGNED_FIXTURE_URL = urljoin(PULP_FIXTURES_BASE_URL, 'drpm-unsigned/')
-"""The URL to a repository with unsigned DRPM packages."""
+PULP_FIXTURES_BASE_URL = config.get_config().get_fixtures_url()
 
 RPM_PACKAGE_CONTENT_NAME = 'rpm.package'