Verify if a manifest already exists locally when syncing a remote repo.

Closes #1047

pulp_container/app/downloaders.py

@@ -10,7 +10,6 @@
 
 from pulpcore.plugin.download import DownloaderFactory, HttpDownloader
 
-
 log = getLogger(__name__)
 
 
@@ -51,6 +50,7 @@ async def _run(self, handle_401=True, extra_data=None):
         if extra_data is not None:
             headers = extra_data.get("headers", headers)
             repo_name = extra_data.get("repo_name", None)
+            check_for_existing_manifest = extra_data.get("check_for_existing_manifest", None)
         this_token = self.registry_auth["bearer"]
         basic_auth = self.registry_auth["basic"]
         auth_headers = self.auth_header(this_token, basic_auth)
@@ -59,7 +59,13 @@ async def _run(self, handle_401=True, extra_data=None):
         self.session._default_auth = None
         if self.download_throttler:
             await self.download_throttler.acquire()
-        async with self.session.get(
+
+        session_method = self.session.get
+
+        if check_for_existing_manifest:
+            session_method = self.session.head
+
+        async with session_method(
             self.url, headers=headers, proxy=self.proxy, proxy_auth=self.proxy_auth
         ) as response:
             try:

pulp_container/app/tasks/sync_stages.py

@@ -114,7 +114,11 @@ async def run(self):
             )
             url = urljoin(self.remote.url, relative_url)
             downloader = self.remote.get_downloader(url=url)
-            to_download.append(downloader.run(extra_data={"headers": V2_ACCEPT_HEADERS}))
+            to_download.append(
+                downloader.run(
+                    extra_data={"headers": V2_ACCEPT_HEADERS, "check_for_existing_manifest": True}
+                )
+            )
 
         async with ProgressReport(
             message="Processing Tags",
@@ -123,13 +127,40 @@ async def run(self):
         ) as pb_parsed_tags:
 
             for download_tag in asyncio.as_completed(to_download):
-                dl_res = await download_tag
-                with open(dl_res.path, "rb") as content_file:
-                    raw_data = content_file.read()
-                dl_res.artifact_attributes["file"] = dl_res.path
-                saved_artifact = await sync_to_async(_save_artifact_blocking)(
-                    dl_res.artifact_attributes
-                )
+                async with ProgressReport(
+                    message="Processing Manifests", code="sync.processing.manifest"
+                ) as pr_manifest:
+                    dl_res = await download_tag
+
+                    digest = dl_res.headers.get("docker-content-digest")
+                    manifest = await sync_to_async(
+                        Manifest.objects.prefetch_related("contentartifact_set")
+                        .filter(digest=digest)
+                        .first
+                    )()
+                    if manifest:
+
+                        def _get_content_data_blocking():
+                            saved_artifact = manifest.contentartifact_set.first().artifact
+                            content_file = saved_artifact.file
+                            raw_data = content_file.read()
+                            return saved_artifact, raw_data
+
+                        saved_artifact, raw_data = await sync_to_async(_get_content_data_blocking)()
+
+                    else:
+                        await pr_manifest.aincrement()
+                        downloader = self.remote.get_downloader(url=dl_res.url)
+                        dl_res = await downloader.run(extra_data={"headers": V2_ACCEPT_HEADERS})
+
+                        with open(dl_res.path, "rb") as content_file:
+                            raw_data = content_file.read()
+                        dl_res.artifact_attributes["file"] = dl_res.path
+
+                        # Attention
+                        saved_artifact = await sync_to_async(_save_artifact_blocking)(
+                            dl_res.artifact_attributes
+                        )
 
                 tag_name = dl_res.url.split("/")[-1]
                 tag_dc = DeclarativeContent(Tag(name=tag_name))

pulp_container/tests/functional/api/test_sync.py

@@ -75,6 +75,18 @@ def test_sync(self):
         repository = repository_api.read(repository.pulp_href)
         self.assertEqual(latest_version_href, repository.latest_version_href)
 
+        # Check if we didn't download
+        # repository = repository_api.create(ContainerContainerRepository(**gen_repo()))
+        # self.addCleanup(repository_api.delete, repository.pulp_href)
+
+        sync_response = repository_api.sync(repository.pulp_href, repository_sync_data)
+        task = monitor_task(sync_response.task)
+        manifest = [
+            report for report in task.progress_reports if report.code == "sync.processing.manifest"
+        ]
+
+        self.assertEqual(manifest[0].done, 0)
+
 
 class SyncInvalidURLTestCase(PulpTestCase):
     """Sync a repository with an invalid url on the Remote."""