Merge pull request pulp#1568 from pavelpicka/5738-mirror-repo

Repository mirror closes: #6033 https://pulp.plan.io/issues/6033
pavelpicka · Feb 12, 2020 · afbb107 · afbb107
2 parents fddc87b + 9713b41
commit afbb107
Show file tree

Hide file tree

Showing 9 changed files with 54 additions and 8 deletions.
diff --git a/CHANGES/5738.feature b/CHANGES/5738.feature
@@ -0,0 +1 @@
+Add mirror mode for sync endpoint.
diff --git a/CHANGES/6033.feature b/CHANGES/6033.feature
@@ -0,0 +1 @@
+SRPM can be skipped during the sync.
diff --git a/docs/_scripts/sync.sh b/docs/_scripts/sync.sh
@@ -3,7 +3,7 @@
 # Sync repository foo using remote bar
 echo "Create a task to sync the repository using the remote."
 export TASK_URL=$(http POST $BASE_ADDR$REPO_HREF'sync/' \
-    remote=$REMOTE_HREF \
+    remote=$REMOTE_HREF skip:="[\"srpm\"]" \
     | jq -r '.task')
 
 # Poll the task (here we use a function defined in docs/_scripts/base.sh)

diff --git a/docs/_static/api.json b/docs/_static/api.json
diff --git a/docs/workflows/create_sync_publish.rst b/docs/workflows/create_sync_publish.rst
@@ -62,9 +62,19 @@ Remote GET response:
 Sync repository ``foo`` using remote ``bar``
 --------------------------------------------
 
+It is possible to skip RPM types by specify list of them in 'skip' parameter.
+
+Supported types to skip are
+
+- SRPM as ``'srpm'``
+
 .. literalinclude:: ../_scripts/sync.sh
    :language: bash
 
+You can specify ``mirror=True`` for a mirror mode. It means Pulp won't update
+repository using previous repository version but create a new copy of remote
+repository as a new repository version.
+
 RepositoryVersion GET response (when sync task complete):
 
 .. code:: json

diff --git a/pulp_rpm/app/constants.py b/pulp_rpm/app/constants.py
@@ -78,6 +78,7 @@
 MODULAR_REPODATA = ['modules']
 COMPS_REPODATA = ['group']
 SKIP_REPODATA = ['group_gz', 'prestodelta']
+OPT_SKIP_TYPES = ['srpm']
 
 CR_UPDATE_RECORD_ATTRS = SimpleNamespace(
     ID='id',

diff --git a/pulp_rpm/app/serializers.py b/pulp_rpm/app/serializers.py
@@ -21,6 +21,7 @@
     RelatedField,
     RemoteSerializer,
     RepositorySerializer,
+    RepositorySyncURLSerializer,
     SingleArtifactContentUploadSerializer,
     validate_unknown_fields,
 )
@@ -37,6 +38,7 @@
     PULP_UPDATE_RECORD_ATTRS,
     PULP_UPDATE_REFERENCE_ATTRS,
     PULP_UPDATE_COLLECTION_ATTRS,
+    OPT_SKIP_TYPES,
     CR_UPDATE_REFERENCE_ATTRS
 )
 
@@ -1103,3 +1105,18 @@ class Meta:
             'data_type', 'checksum_type', 'checksum'
         )
         model = RepoMetadataFile
+
+
+class RpmRepositorySyncURLSerializer(RepositorySyncURLSerializer):
+    """
+    A serializer for Repository Sync.
+    """
+
+    skip = serializers.ListField(
+        help_text=_("List of content types to skip during sync."),
+        required=False,
+        default=[],
+        child=serializers.ChoiceField(
+            [(name, name) for name in OPT_SKIP_TYPES]
+        )
+    )
diff --git a/pulp_rpm/app/tasks/synchronizing.py b/pulp_rpm/app/tasks/synchronizing.py
@@ -97,15 +97,17 @@ def repodata_exists(remote, url):
     return True
 
 
-def synchronize(remote_pk, repository_pk):
+def synchronize(remote_pk, repository_pk, mirror, skip):
     """
     Sync content from the remote repository.
 
     Create a new version of the repository that is synchronized with the remote.
 
     Args:
+        mirror (bool): Mirror mode
         remote_pk (str): The remote PK.
         repository_pk (str): The repository PK.
+        skip (list): List of content types to skip.
 
     Raises:
         ValueError: If the remote does not specify a url to sync.
@@ -144,9 +146,10 @@ def synchronize(remote_pk, repository_pk):
                                            repository=sub_repo)
                 dv.create()
 
-    first_stage = RpmFirstStage(remote, deferred_download, treeinfo=treeinfo)
+    first_stage = RpmFirstStage(remote, deferred_download, treeinfo=treeinfo, skip=skip)
     dv = RpmDeclarativeVersion(first_stage=first_stage,
-                               repository=repository)
+                               repository=repository,
+                               mirror=mirror)
     dv.create()
 
 
@@ -189,7 +192,7 @@ class RpmFirstStage(Stage):
     that should exist in the new :class:`~pulpcore.plugin.models.RepositoryVersion`.
     """
 
-    def __init__(self, remote, deferred_download, new_url=None, treeinfo=None):
+    def __init__(self, remote, deferred_download, new_url=None, treeinfo=None, skip=None):
         """
         The first stage of a pulp_rpm sync pipeline.
 
@@ -201,13 +204,15 @@ def __init__(self, remote, deferred_download, new_url=None, treeinfo=None):
         Keyword Args:
             new_url(str): URL to replace remote url
             treeinfo(dict): Treeinfo data
+            skip(list): List of RPM types to skip
 
         """
         super().__init__()
         self.remote = remote
         self.deferred_download = deferred_download
         self.new_url = new_url
         self.treeinfo = treeinfo
+        self.skip = skip
 
     @staticmethod
     async def parse_updateinfo(updateinfo_xml_path):
@@ -586,6 +591,12 @@ async def run(self):
                                                                       filelists_xml_path,
                                                                       other_xml_path)
 
+                        # avoid SRPM if defined by user
+                        if 'srpm' in self.skip:
+                            packages = {
+                                pkgId: pkg for pkgId, pkg in packages.items() if pkg.arch != 'src'
+                            }
+
                         progress_data = {
                             'message': 'Parsed Packages',
                             'code': 'parsing.packages',

diff --git a/pulp_rpm/app/viewsets.py b/pulp_rpm/app/viewsets.py
@@ -54,6 +54,7 @@
     RpmDistributionSerializer,
     RpmRemoteSerializer,
     RpmRepositorySerializer,
+    RpmRepositorySyncURLSerializer,
     RpmPublicationSerializer,
     UpdateRecordSerializer,
 )
@@ -115,19 +116,23 @@ def sync(self, request, pk):
         Dispatches a sync task.
         """
         repository = self.get_object()
-        serializer = RepositorySyncURLSerializer(
+        serializer = RpmRepositorySyncURLSerializer(
             data=request.data,
             context={'request': request}
         )
         serializer.is_valid(raise_exception=True)
         remote = serializer.validated_data.get('remote')
+        mirror = serializer.validated_data.get('mirror')
+        skip = serializer.validated_data.get('skip')
 
         result = enqueue_with_reservation(
             tasks.synchronize,
             [repository, remote],
             kwargs={
+                'mirror': mirror,
                 'remote_pk': remote.pk,
-                'repository_pk': repository.pk
+                'repository_pk': repository.pk,
+                'skip': skip
             }
         )
         return OperationPostponedResponse(result, request)