Add lazy sync support

Add support for lazy download policies Update _gen_verbose_remote to add download policy field. Add constant DOWNLOAD_POLICIES. closes #4212 https://pulp.plan.io/issues/4212 re #4182 https://pulp.plan.io/issues/4182
pulp · Jan 18, 2019 · 9590647 · 9590647
1 parent 08caeb4
commit 9590647
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 10 deletions.
diff --git a/docs/workflows/create_sync_publish.rst b/docs/workflows/create_sync_publish.rst
@@ -21,7 +21,7 @@ Create a repository ``foo``
 Create a new remote ``bar``
 ---------------------------
 
-``$ http POST http://localhost:8000/pulp/api/v3/remotes/rpm/ name='bar' url='https://repos.fedorapeople.org/pulp/pulp/fixtures/rpm-unsigned/'``
+``$ http POST http://localhost:8000/pulp/api/v3/remotes/rpm/ name='bar' url='https://repos.fedorapeople.org/pulp/pulp/fixtures/rpm-unsigned/' policy='on_demand'``
 
 .. code:: json
 
@@ -30,8 +30,7 @@ Create a new remote ``bar``
         ...
     }
 
-``$ export REMOTE_HREF=$(http :8000/pulp/api/v3/remotes/rpm/ | jq -r '.results[] | select(.name ==
-"bar") | ._href')``
+``$ export REMOTE_HREF=$(http :8000/pulp/api/v3/remotes/rpm/ | jq -r '.results[] | select(.name == "bar") | ._href')``
 
 Sync repository ``foo`` using remote ``bar``
 --------------------------------------------

diff --git a/pulp_rpm/app/tasks/synchronizing.py b/pulp_rpm/app/tasks/synchronizing.py
@@ -7,7 +7,7 @@
 
 import createrepo_c as cr
 
-from pulpcore.plugin.models import Artifact, ProgressBar, Repository, RepositoryVersion
+from pulpcore.plugin.models import Artifact, ProgressBar, Remote, Repository, RepositoryVersion
 from pulpcore.plugin.stages import (
     DeclarativeArtifact,
     DeclarativeContent,
@@ -59,17 +59,21 @@ def synchronize(remote_pk, repository_pk):
     log.info(_('Synchronizing: repository={r} remote={p}').format(
         r=repository.name, p=remote.name))
 
+    download_artifacts = (remote.policy == Remote.IMMEDIATE)
     first_stage = RpmFirstStage(remote)
     with WorkingDirectory():
         with RepositoryVersion.create(repository) as new_version:
             loop = asyncio.get_event_loop()
             remove_duplicates_stage = RemoveDuplicates(new_version, **dupe_criteria)
-            stages = [
-                first_stage,
-                QueryExistingArtifacts(), ArtifactDownloader(), ArtifactSaver(),
+            stages = [first_stage]
+
+            if download_artifacts:
+                stages.extend([QueryExistingArtifacts(), ArtifactDownloader(), ArtifactSaver()])
+
+            stages.extend([
                 QueryExistingContentUnits(), ErratumContentUnitSaver(), remove_duplicates_stage,
                 ContentUnitAssociation(new_version), EndStage()
-            ]
+            ])
             pipeline = create_pipeline(stages)
             loop.run_until_complete(pipeline)
 
@@ -192,8 +196,9 @@ async def __call__(self, in_q, out_q):
 
         """
         with ProgressBar(message='Downloading and Parsing Metadata') as pb:
-            downloader = self.remote.get_downloader(url=urljoin(self.remote.url,
-                                                                'repodata/repomd.xml'))
+            downloader = self.remote.get_downloader(
+                url=urljoin(self.remote.url, 'repodata/repomd.xml')
+            )
             # TODO: decide how to distinguish between a mirror list and a normal repo
             result = await downloader.run()
             pb.increment()

diff --git a/pulp_rpm/tests/functional/api/test_crud_remotes.py b/pulp_rpm/tests/functional/api/test_crud_remotes.py
@@ -6,6 +6,7 @@
 from requests.exceptions import HTTPError
 
 from pulp_smash import api, config, utils
+from pulp_smash.pulp3.constants import DOWNLOAD_POLICIES
 
 from pulp_rpm.tests.functional.constants import RPM_REMOTE_PATH
 from pulp_rpm.tests.functional.utils import gen_rpm_remote, skip_if
@@ -126,6 +127,7 @@ def _gen_verbose_remote():
     attrs.update({
         'password': utils.uuid4(),
         'username': utils.uuid4(),
+        'policy': choice(DOWNLOAD_POLICIES),
         'validate': choice((False, True)),
     })
     return attrs