Fix syncs from file:// repos and add a test

closes: #9021
https://pulp.plan.io/issues/9021
closes: #7350
https://pulp.plan.io/issues/7350

coverage.md

@@ -16,13 +16,14 @@ This file contains list of features and their test coverage.
   As a user, I can sync all yum content types ( NO drpm) with optimization in mirror mode| PART
    | only types contained in rpm-unsigned |
 | As a user, my mirror-mode syncs are exactly identical to the upstream repo (checksums, metadata files, repomd signature, package locations, extra_files.json) | NO | Needs fixture https://pulp.plan.io/issues/8809 |
+| As a user, I can sync an RPM repository from the local filesystem | PART | Only tested with basic fixture in immediate mode with mirroring enabled |
 | As a user, I can sync and skip specific type (srpm) | YES |  |
 | As a user, I can sync opensuse repository | NO |  |
 | As a user, I can sync Oracle repository using ULN | NO | |
 | As a user, I can sync from a mirror list | YES |  |
 | As a user, I can sync from a mirror list with comments | YES |  |
 | As a user, I can sync from CDN using certificates | YES |  |
-| As a user, I can re-sync custom reposotory metadata when it was the only change in a repository | YES |  |
+| As a user, I can re-sync custom repository metadata when it was the only change in a repository | YES |  |
 | **Duplicates** |  |  |
 | As a user, I have only one advisory with the same id in a repo version | YES |  |
 | As a user, I have only one module with the same NSVCA in a repo version | NO |  |

pulp_rpm/app/tasks/synchronizing.py

@@ -633,6 +633,8 @@ async def run_repomdrecord_download(name, location_href, downloader):
                         metadata_pb.increment()
                 except ClientResponseError as exc:
                     raise HTTPNotFound(reason=_("File not found: {}".format(exc.request_info.url)))
+                except FileNotFoundError:
+                    raise
 
                 if self.mirror:
                     # optional signature and key files for repomd metadata
@@ -644,7 +646,7 @@ async def run_repomdrecord_download(name, location_href, downloader):
                             result = await downloader.run()
                             store_metadata_for_mirroring(self.repository, result, file_href)
                             metadata_pb.increment()
-                        except ClientResponseError:
+                        except (ClientResponseError, FileNotFoundError):
                             pass
 
                     # extra files to copy, e.g. EULA, LICENSE
@@ -655,7 +657,7 @@ async def run_repomdrecord_download(name, location_href, downloader):
                         result = await downloader.run()
                         store_metadata_for_mirroring(self.repository, result, "extra_files.json")
                         metadata_pb.increment()
-                    except ClientResponseError:
+                    except (ClientResponseError, FileNotFoundError):
                         pass
                     else:
                         try:
@@ -676,6 +678,8 @@ async def run_repomdrecord_download(name, location_href, downloader):
                             raise HTTPNotFound(
                                 reason=_("File not found: {}".format(exc.request_info.url))
                             )
+                        except FileNotFoundError:
+                            raise
 
             await self.parse_repository_metadata(repomd, repomd_files, file_extension)
 

pulp_rpm/tests/functional/api/test_sync.py

@@ -128,6 +128,29 @@ def test_sync(self):
         self.assertEqual(latest_version_href, repo.latest_version_href)
         self.assertDictEqual(get_content_summary(repo.to_dict()), RPM_FIXTURE_SUMMARY)
 
+    def test_sync_local(self):
+        """Test syncing from the local filesystem."""
+        import shutil
+
+        if not shutil.which("wget"):
+            unittest.skip("Cannot run file:// sync tests without 'wget' available.")
+
+        cli.Client(self.cfg).run(
+            (
+                "wget",
+                "--recursive",
+                "--no-parent",
+                "--no-host-directories",
+                "--directory-prefix",
+                "/tmp",
+                RPM_UNSIGNED_FIXTURE_URL,
+            )
+        )
+
+        remote = self.remote_api.create(gen_rpm_remote(url="file:///tmp/rpm-unsigned/"))
+
+        self.do_test(remote=remote, mirror=True)
+
     def test_sync_from_valid_mirror_list_feed(self):
         """Sync RPM content from a mirror list feed which contains a valid remote URL."""
         remote = self.remote_api.create(gen_rpm_remote(RPM_MIRROR_LIST_GOOD_FIXTURE_URL))
@@ -1094,14 +1117,15 @@ def test_sync_skip_srpm(self):
         self.assertEqual(present_package_count, 0)
         self.assertEqual(present_advisory_count, SRPM_UNSIGNED_FIXTURE_ADVISORY_COUNT)
 
-    def do_test(self, repository=None, remote=None):
+    def do_test(self, repository=None, remote=None, mirror=False):
         """Sync a repository.
 
         Args:
             repository (pulp_rpm.app.models.repository.RpmRepository):
                 object of RPM repository
             remote (pulp_rpm.app.models.repository.RpmRemote):
                 object of RPM Remote
+            mirror (bool): Whether to use mirror-mode during the sync
         Returns (tuple):
             tuple of instances of
             pulp_rpm.app.models.repository.RpmRepository, pulp_rpm.app.models.repository.RpmRemote
@@ -1118,7 +1142,7 @@ def do_test(self, repository=None, remote=None):
         else:
             remote = self.remote_api.read(remote.pulp_href)
 
-        repository_sync_data = RpmRepositorySyncURL(remote=remote.pulp_href)
+        repository_sync_data = RpmRepositorySyncURL(remote=remote.pulp_href, mirror=mirror)
         sync_response = self.repo_api.sync(repo.pulp_href, repository_sync_data)
         monitor_task(sync_response.task)
         return self.repo_api.read(repo.pulp_href), self.remote_api.read(remote.pulp_href)